2024-11-07 12:40:51,196 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-07 12:40:51,213 main DEBUG Took 0.014153 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-07 12:40:51,213 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-07 12:40:51,214 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-07 12:40:51,215 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-07 12:40:51,216 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 12:40:51,226 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-07 12:40:51,243 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 12:40:51,244 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 12:40:51,245 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 12:40:51,246 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 12:40:51,246 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 12:40:51,247 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 12:40:51,248 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 12:40:51,248 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 12:40:51,249 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 12:40:51,249 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 12:40:51,250 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 12:40:51,251 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 12:40:51,251 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 12:40:51,252 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 12:40:51,252 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 12:40:51,253 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 12:40:51,253 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 12:40:51,254 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 12:40:51,254 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 12:40:51,255 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 12:40:51,255 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 12:40:51,256 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 12:40:51,256 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 12:40:51,258 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 12:40:51,260 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 12:40:51,260 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-07 12:40:51,263 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 12:40:51,265 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-07 12:40:51,267 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-07 12:40:51,267 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-07 12:40:51,269 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-07 12:40:51,269 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-07 12:40:51,280 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-07 12:40:51,284 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-07 12:40:51,289 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-07 12:40:51,290 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-07 12:40:51,291 main DEBUG createAppenders(={Console}) 2024-11-07 12:40:51,294 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-07 12:40:51,295 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-07 12:40:51,296 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-07 12:40:51,297 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-07 12:40:51,298 main DEBUG OutputStream closed 2024-11-07 12:40:51,299 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-07 12:40:51,300 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-07 12:40:51,300 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-07 12:40:51,436 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-07 12:40:51,439 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-07 12:40:51,440 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-07 12:40:51,441 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-07 12:40:51,442 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-07 12:40:51,443 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-07 12:40:51,443 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-07 12:40:51,444 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-07 12:40:51,444 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-07 12:40:51,444 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-07 12:40:51,445 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-07 12:40:51,446 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-07 12:40:51,447 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-07 12:40:51,447 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-07 12:40:51,447 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-07 12:40:51,448 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-07 12:40:51,448 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-07 12:40:51,449 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-07 12:40:51,451 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-07 12:40:51,452 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-07 12:40:51,452 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-07 12:40:51,453 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-07T12:40:51,844 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ba7e58d5-de0c-7392-0264-ead072f99c84 2024-11-07 12:40:51,848 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-07 12:40:51,849 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-07T12:40:51,862 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-07T12:40:51,890 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-07T12:40:51,919 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ba7e58d5-de0c-7392-0264-ead072f99c84/cluster_73ed722e-8252-77e5-7683-f63666bd3476, deleteOnExit=true 2024-11-07T12:40:51,919 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-07T12:40:51,921 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ba7e58d5-de0c-7392-0264-ead072f99c84/test.cache.data in system properties and HBase conf 2024-11-07T12:40:51,921 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ba7e58d5-de0c-7392-0264-ead072f99c84/hadoop.tmp.dir in system properties and HBase conf 2024-11-07T12:40:51,922 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ba7e58d5-de0c-7392-0264-ead072f99c84/hadoop.log.dir in system properties and HBase conf 2024-11-07T12:40:51,923 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ba7e58d5-de0c-7392-0264-ead072f99c84/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-07T12:40:51,923 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ba7e58d5-de0c-7392-0264-ead072f99c84/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-07T12:40:51,924 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-07T12:40:52,068 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-07T12:40:52,213 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-07T12:40:52,218 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ba7e58d5-de0c-7392-0264-ead072f99c84/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-07T12:40:52,219 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ba7e58d5-de0c-7392-0264-ead072f99c84/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-07T12:40:52,220 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ba7e58d5-de0c-7392-0264-ead072f99c84/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-07T12:40:52,220 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ba7e58d5-de0c-7392-0264-ead072f99c84/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-07T12:40:52,221 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ba7e58d5-de0c-7392-0264-ead072f99c84/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-07T12:40:52,222 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ba7e58d5-de0c-7392-0264-ead072f99c84/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-07T12:40:52,222 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ba7e58d5-de0c-7392-0264-ead072f99c84/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-07T12:40:52,223 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ba7e58d5-de0c-7392-0264-ead072f99c84/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-07T12:40:52,223 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ba7e58d5-de0c-7392-0264-ead072f99c84/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-07T12:40:52,224 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ba7e58d5-de0c-7392-0264-ead072f99c84/nfs.dump.dir in system properties and HBase conf 2024-11-07T12:40:52,226 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ba7e58d5-de0c-7392-0264-ead072f99c84/java.io.tmpdir in system properties and HBase conf 2024-11-07T12:40:52,226 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ba7e58d5-de0c-7392-0264-ead072f99c84/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-07T12:40:52,227 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ba7e58d5-de0c-7392-0264-ead072f99c84/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-07T12:40:52,227 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ba7e58d5-de0c-7392-0264-ead072f99c84/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-07T12:40:53,248 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-07T12:40:53,352 INFO [Time-limited test {}] log.Log(170): Logging initialized @3257ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-07T12:40:53,456 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-07T12:40:53,569 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-07T12:40:53,619 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-07T12:40:53,620 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-07T12:40:53,622 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-07T12:40:53,645 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-07T12:40:53,652 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ba7e58d5-de0c-7392-0264-ead072f99c84/hadoop.log.dir/,AVAILABLE} 2024-11-07T12:40:53,655 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-07T12:40:53,920 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ba7e58d5-de0c-7392-0264-ead072f99c84/java.io.tmpdir/jetty-localhost-45309-hadoop-hdfs-3_4_1-tests_jar-_-any-16666321968129654234/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-07T12:40:53,953 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:45309} 2024-11-07T12:40:53,953 INFO [Time-limited test {}] server.Server(415): Started @3860ms 2024-11-07T12:40:54,498 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-07T12:40:54,511 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-07T12:40:54,517 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-07T12:40:54,518 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-07T12:40:54,518 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-07T12:40:54,521 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ba7e58d5-de0c-7392-0264-ead072f99c84/hadoop.log.dir/,AVAILABLE} 2024-11-07T12:40:54,522 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-07T12:40:54,666 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ba7e58d5-de0c-7392-0264-ead072f99c84/java.io.tmpdir/jetty-localhost-39059-hadoop-hdfs-3_4_1-tests_jar-_-any-12944993326540046936/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-07T12:40:54,667 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:39059} 2024-11-07T12:40:54,667 INFO [Time-limited test {}] server.Server(415): Started @4574ms 2024-11-07T12:40:54,728 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-07T12:40:57,659 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ba7e58d5-de0c-7392-0264-ead072f99c84/cluster_73ed722e-8252-77e5-7683-f63666bd3476/data/data2/current/BP-138800792-172.17.0.2-1730983252949/current, will proceed with Du for space computation calculation, 2024-11-07T12:40:57,661 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ba7e58d5-de0c-7392-0264-ead072f99c84/cluster_73ed722e-8252-77e5-7683-f63666bd3476/data/data1/current/BP-138800792-172.17.0.2-1730983252949/current, will proceed with Du for space computation calculation, 2024-11-07T12:40:57,730 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-07T12:40:57,818 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x44cc985f71ec1b6c with lease ID 0x8c786926b9f1ceb6: Processing first storage report for DS-bd733cd2-30f1-4541-8f71-f5cc8e9227da from datanode DatanodeRegistration(127.0.0.1:36063, datanodeUuid=bea1d833-e7f4-4776-b040-860a71f94e0f, infoPort=40023, infoSecurePort=0, ipcPort=42939, storageInfo=lv=-57;cid=testClusterID;nsid=1715072385;c=1730983252950) 2024-11-07T12:40:57,819 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x44cc985f71ec1b6c with lease ID 0x8c786926b9f1ceb6: from storage DS-bd733cd2-30f1-4541-8f71-f5cc8e9227da node DatanodeRegistration(127.0.0.1:36063, datanodeUuid=bea1d833-e7f4-4776-b040-860a71f94e0f, infoPort=40023, infoSecurePort=0, ipcPort=42939, storageInfo=lv=-57;cid=testClusterID;nsid=1715072385;c=1730983252950), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-07T12:40:57,819 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x44cc985f71ec1b6c with lease ID 0x8c786926b9f1ceb6: Processing first storage report for DS-b3c9788d-1eed-42dd-ac40-b696f8120247 from datanode DatanodeRegistration(127.0.0.1:36063, datanodeUuid=bea1d833-e7f4-4776-b040-860a71f94e0f, infoPort=40023, infoSecurePort=0, ipcPort=42939, storageInfo=lv=-57;cid=testClusterID;nsid=1715072385;c=1730983252950) 2024-11-07T12:40:57,820 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x44cc985f71ec1b6c with lease ID 0x8c786926b9f1ceb6: from storage DS-b3c9788d-1eed-42dd-ac40-b696f8120247 node DatanodeRegistration(127.0.0.1:36063, datanodeUuid=bea1d833-e7f4-4776-b040-860a71f94e0f, infoPort=40023, infoSecurePort=0, ipcPort=42939, storageInfo=lv=-57;cid=testClusterID;nsid=1715072385;c=1730983252950), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-07T12:40:57,841 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ba7e58d5-de0c-7392-0264-ead072f99c84 2024-11-07T12:40:57,958 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ba7e58d5-de0c-7392-0264-ead072f99c84/cluster_73ed722e-8252-77e5-7683-f63666bd3476/zookeeper_0, clientPort=49805, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ba7e58d5-de0c-7392-0264-ead072f99c84/cluster_73ed722e-8252-77e5-7683-f63666bd3476/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ba7e58d5-de0c-7392-0264-ead072f99c84/cluster_73ed722e-8252-77e5-7683-f63666bd3476/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-07T12:40:57,974 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=49805 2024-11-07T12:40:57,991 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T12:40:57,995 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T12:40:58,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741825_1001 (size=7) 2024-11-07T12:40:58,897 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4 with version=8 2024-11-07T12:40:58,898 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/hbase-staging 2024-11-07T12:40:59,039 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-07T12:40:59,406 INFO [Time-limited test {}] client.ConnectionUtils(128): master/db9ad1cb6cf9:0 server-side Connection retries=45 2024-11-07T12:40:59,417 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T12:40:59,418 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-07T12:40:59,422 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-07T12:40:59,422 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T12:40:59,422 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-07T12:40:59,652 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-07T12:40:59,723 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-07T12:40:59,736 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-07T12:40:59,742 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-07T12:40:59,782 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 1044 (auto-detected) 2024-11-07T12:40:59,783 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-07T12:40:59,809 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38619 2024-11-07T12:40:59,841 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:38619 connecting to ZooKeeper ensemble=127.0.0.1:49805 2024-11-07T12:40:59,887 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:386190x0, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-07T12:40:59,894 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:38619-0x1001a4271e00000 connected 2024-11-07T12:40:59,950 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T12:40:59,953 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T12:40:59,981 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38619-0x1001a4271e00000, quorum=127.0.0.1:49805, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T12:40:59,988 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4, hbase.cluster.distributed=false 2024-11-07T12:41:00,053 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38619-0x1001a4271e00000, quorum=127.0.0.1:49805, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-07T12:41:00,061 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38619 2024-11-07T12:41:00,064 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38619 2024-11-07T12:41:00,076 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38619 2024-11-07T12:41:00,088 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38619 2024-11-07T12:41:00,100 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38619 2024-11-07T12:41:00,270 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/db9ad1cb6cf9:0 server-side Connection retries=45 2024-11-07T12:41:00,273 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T12:41:00,274 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-07T12:41:00,274 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-07T12:41:00,274 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T12:41:00,274 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-07T12:41:00,278 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-07T12:41:00,282 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-07T12:41:00,283 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34293 2024-11-07T12:41:00,286 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34293 connecting to ZooKeeper ensemble=127.0.0.1:49805 2024-11-07T12:41:00,288 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T12:41:00,292 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T12:41:00,304 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:342930x0, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-07T12:41:00,305 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:342930x0, quorum=127.0.0.1:49805, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T12:41:00,310 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-07T12:41:00,316 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34293-0x1001a4271e00001 connected 2024-11-07T12:41:00,325 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-07T12:41:00,327 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34293-0x1001a4271e00001, quorum=127.0.0.1:49805, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-07T12:41:00,333 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34293-0x1001a4271e00001, quorum=127.0.0.1:49805, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-07T12:41:00,334 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34293 2024-11-07T12:41:00,335 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34293 2024-11-07T12:41:00,335 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34293 2024-11-07T12:41:00,336 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34293 2024-11-07T12:41:00,336 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34293 2024-11-07T12:41:00,362 DEBUG [M:0;db9ad1cb6cf9:38619 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;db9ad1cb6cf9:38619 2024-11-07T12:41:00,364 INFO [master/db9ad1cb6cf9:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/db9ad1cb6cf9,38619,1730983259119 2024-11-07T12:41:00,373 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34293-0x1001a4271e00001, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T12:41:00,374 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38619-0x1001a4271e00000, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T12:41:00,376 DEBUG [master/db9ad1cb6cf9:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38619-0x1001a4271e00000, quorum=127.0.0.1:49805, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/db9ad1cb6cf9,38619,1730983259119 2024-11-07T12:41:00,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34293-0x1001a4271e00001, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-07T12:41:00,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38619-0x1001a4271e00000, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T12:41:00,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34293-0x1001a4271e00001, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T12:41:00,408 DEBUG [master/db9ad1cb6cf9:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38619-0x1001a4271e00000, quorum=127.0.0.1:49805, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-07T12:41:00,410 INFO [master/db9ad1cb6cf9:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/db9ad1cb6cf9,38619,1730983259119 from backup master directory 2024-11-07T12:41:00,414 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34293-0x1001a4271e00001, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T12:41:00,414 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38619-0x1001a4271e00000, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/db9ad1cb6cf9,38619,1730983259119 2024-11-07T12:41:00,414 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38619-0x1001a4271e00000, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T12:41:00,416 WARN [master/db9ad1cb6cf9:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-07T12:41:00,416 INFO [master/db9ad1cb6cf9:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=db9ad1cb6cf9,38619,1730983259119 2024-11-07T12:41:00,419 INFO [master/db9ad1cb6cf9:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-07T12:41:00,426 INFO [master/db9ad1cb6cf9:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-07T12:41:00,508 DEBUG [master/db9ad1cb6cf9:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/hbase.id] with ID: 2e542096-4b0e-4b01-9952-662501fc0f65 2024-11-07T12:41:00,509 DEBUG [master/db9ad1cb6cf9:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/.tmp/hbase.id 2024-11-07T12:41:00,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741826_1002 (size=42) 2024-11-07T12:41:00,929 DEBUG [master/db9ad1cb6cf9:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/.tmp/hbase.id]:[hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/hbase.id] 2024-11-07T12:41:01,011 INFO [master/db9ad1cb6cf9:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T12:41:01,018 INFO [master/db9ad1cb6cf9:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-07T12:41:01,044 INFO [master/db9ad1cb6cf9:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 23ms. 2024-11-07T12:41:01,048 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38619-0x1001a4271e00000, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T12:41:01,049 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34293-0x1001a4271e00001, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T12:41:01,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741827_1003 (size=196) 2024-11-07T12:41:01,107 INFO [master/db9ad1cb6cf9:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T12:41:01,111 INFO [master/db9ad1cb6cf9:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-07T12:41:01,137 DEBUG [master/db9ad1cb6cf9:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:01,145 INFO [master/db9ad1cb6cf9:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-07T12:41:01,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741828_1004 (size=1189) 2024-11-07T12:41:01,217 INFO [master/db9ad1cb6cf9:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/MasterData/data/master/store 2024-11-07T12:41:01,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741829_1005 (size=34) 2024-11-07T12:41:01,251 INFO [master/db9ad1cb6cf9:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-07T12:41:01,258 DEBUG [master/db9ad1cb6cf9:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:41:01,259 DEBUG [master/db9ad1cb6cf9:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-07T12:41:01,260 INFO [master/db9ad1cb6cf9:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T12:41:01,260 DEBUG [master/db9ad1cb6cf9:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T12:41:01,263 DEBUG [master/db9ad1cb6cf9:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-07T12:41:01,263 DEBUG [master/db9ad1cb6cf9:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T12:41:01,263 INFO [master/db9ad1cb6cf9:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T12:41:01,265 DEBUG [master/db9ad1cb6cf9:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1730983261259Disabling compacts and flushes for region at 1730983261259Disabling writes for close at 1730983261263 (+4 ms)Writing region close event to WAL at 1730983261263Closed at 1730983261263 2024-11-07T12:41:01,269 WARN [master/db9ad1cb6cf9:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/MasterData/data/master/store/.initializing 2024-11-07T12:41:01,269 DEBUG [master/db9ad1cb6cf9:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/MasterData/WALs/db9ad1cb6cf9,38619,1730983259119 2024-11-07T12:41:01,283 INFO [master/db9ad1cb6cf9:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-07T12:41:01,306 INFO [master/db9ad1cb6cf9:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=db9ad1cb6cf9%2C38619%2C1730983259119, suffix=, logDir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/MasterData/WALs/db9ad1cb6cf9,38619,1730983259119, archiveDir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/MasterData/oldWALs, maxLogs=10 2024-11-07T12:41:01,342 DEBUG [master/db9ad1cb6cf9:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/MasterData/WALs/db9ad1cb6cf9,38619,1730983259119/db9ad1cb6cf9%2C38619%2C1730983259119.1730983261313, exclude list is [], retry=0 2024-11-07T12:41:01,370 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36063,DS-bd733cd2-30f1-4541-8f71-f5cc8e9227da,DISK] 2024-11-07T12:41:01,375 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-07T12:41:01,430 INFO [master/db9ad1cb6cf9:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/MasterData/WALs/db9ad1cb6cf9,38619,1730983259119/db9ad1cb6cf9%2C38619%2C1730983259119.1730983261313 2024-11-07T12:41:01,431 DEBUG [master/db9ad1cb6cf9:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40023:40023)] 2024-11-07T12:41:01,432 DEBUG [master/db9ad1cb6cf9:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-07T12:41:01,433 DEBUG [master/db9ad1cb6cf9:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:41:01,437 DEBUG [master/db9ad1cb6cf9:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T12:41:01,438 DEBUG [master/db9ad1cb6cf9:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T12:41:01,503 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-07T12:41:01,537 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-07T12:41:01,542 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:01,548 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T12:41:01,549 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-07T12:41:01,560 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-07T12:41:01,560 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:01,562 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:41:01,562 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-07T12:41:01,567 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-07T12:41:01,567 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:01,572 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:41:01,573 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-07T12:41:01,579 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-07T12:41:01,579 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:01,581 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:41:01,581 DEBUG [master/db9ad1cb6cf9:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T12:41:01,586 DEBUG [master/db9ad1cb6cf9:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-07T12:41:01,587 DEBUG [master/db9ad1cb6cf9:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-07T12:41:01,597 DEBUG [master/db9ad1cb6cf9:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T12:41:01,598 DEBUG [master/db9ad1cb6cf9:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T12:41:01,603 DEBUG [master/db9ad1cb6cf9:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-07T12:41:01,611 DEBUG [master/db9ad1cb6cf9:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T12:41:01,616 DEBUG [master/db9ad1cb6cf9:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T12:41:01,618 INFO [master/db9ad1cb6cf9:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67437138, jitterRate=0.0048916637897491455}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-07T12:41:01,628 DEBUG [master/db9ad1cb6cf9:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1730983261455Initializing all the Stores at 1730983261460 (+5 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730983261460Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730983261464 (+4 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730983261465 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730983261465Cleaning up temporary data from old regions at 1730983261599 (+134 ms)Region opened successfully at 1730983261628 (+29 ms) 2024-11-07T12:41:01,631 INFO [master/db9ad1cb6cf9:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-07T12:41:01,681 DEBUG [master/db9ad1cb6cf9:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7995748f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=db9ad1cb6cf9/172.17.0.2:0 2024-11-07T12:41:01,717 INFO [master/db9ad1cb6cf9:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-07T12:41:01,730 INFO [master/db9ad1cb6cf9:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-07T12:41:01,730 INFO [master/db9ad1cb6cf9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-07T12:41:01,733 INFO [master/db9ad1cb6cf9:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-07T12:41:01,734 INFO [master/db9ad1cb6cf9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-07T12:41:01,739 INFO [master/db9ad1cb6cf9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-07T12:41:01,739 INFO [master/db9ad1cb6cf9:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-07T12:41:01,807 INFO [master/db9ad1cb6cf9:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-07T12:41:01,824 DEBUG [master/db9ad1cb6cf9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38619-0x1001a4271e00000, quorum=127.0.0.1:49805, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-07T12:41:01,830 DEBUG [master/db9ad1cb6cf9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-07T12:41:01,846 INFO [master/db9ad1cb6cf9:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-07T12:41:01,851 DEBUG [master/db9ad1cb6cf9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38619-0x1001a4271e00000, quorum=127.0.0.1:49805, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-07T12:41:01,854 DEBUG [master/db9ad1cb6cf9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-07T12:41:01,859 INFO [master/db9ad1cb6cf9:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-07T12:41:01,871 DEBUG [master/db9ad1cb6cf9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38619-0x1001a4271e00000, quorum=127.0.0.1:49805, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-07T12:41:01,873 DEBUG [master/db9ad1cb6cf9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-07T12:41:01,876 DEBUG [master/db9ad1cb6cf9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38619-0x1001a4271e00000, quorum=127.0.0.1:49805, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-07T12:41:01,879 DEBUG [master/db9ad1cb6cf9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-07T12:41:01,906 DEBUG [master/db9ad1cb6cf9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38619-0x1001a4271e00000, quorum=127.0.0.1:49805, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-07T12:41:01,910 DEBUG [master/db9ad1cb6cf9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-07T12:41:01,918 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34293-0x1001a4271e00001, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-07T12:41:01,918 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38619-0x1001a4271e00000, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-07T12:41:01,918 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34293-0x1001a4271e00001, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T12:41:01,919 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38619-0x1001a4271e00000, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T12:41:01,923 INFO [master/db9ad1cb6cf9:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=db9ad1cb6cf9,38619,1730983259119, sessionid=0x1001a4271e00000, setting cluster-up flag (Was=false) 2024-11-07T12:41:01,939 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34293-0x1001a4271e00001, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T12:41:01,939 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38619-0x1001a4271e00000, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T12:41:01,946 DEBUG [master/db9ad1cb6cf9:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-07T12:41:01,948 DEBUG [master/db9ad1cb6cf9:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=db9ad1cb6cf9,38619,1730983259119 2024-11-07T12:41:01,955 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34293-0x1001a4271e00001, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T12:41:01,956 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38619-0x1001a4271e00000, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T12:41:01,962 DEBUG [master/db9ad1cb6cf9:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-07T12:41:01,966 DEBUG [master/db9ad1cb6cf9:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=db9ad1cb6cf9,38619,1730983259119 2024-11-07T12:41:01,974 INFO [master/db9ad1cb6cf9:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-07T12:41:02,054 INFO [RS:0;db9ad1cb6cf9:34293 {}] regionserver.HRegionServer(746): ClusterId : 2e542096-4b0e-4b01-9952-662501fc0f65 2024-11-07T12:41:02,061 DEBUG [RS:0;db9ad1cb6cf9:34293 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-07T12:41:02,095 DEBUG [RS:0;db9ad1cb6cf9:34293 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-07T12:41:02,095 DEBUG [RS:0;db9ad1cb6cf9:34293 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-07T12:41:02,100 DEBUG [RS:0;db9ad1cb6cf9:34293 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-07T12:41:02,101 DEBUG [RS:0;db9ad1cb6cf9:34293 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6605bc14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=db9ad1cb6cf9/172.17.0.2:0 2024-11-07T12:41:02,094 DEBUG [master/db9ad1cb6cf9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-07T12:41:02,127 INFO [master/db9ad1cb6cf9:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-07T12:41:02,130 DEBUG [RS:0;db9ad1cb6cf9:34293 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;db9ad1cb6cf9:34293 2024-11-07T12:41:02,138 INFO [master/db9ad1cb6cf9:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-07T12:41:02,144 INFO [RS:0;db9ad1cb6cf9:34293 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-07T12:41:02,145 INFO [RS:0;db9ad1cb6cf9:34293 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-07T12:41:02,145 DEBUG [RS:0;db9ad1cb6cf9:34293 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-07T12:41:02,149 INFO [RS:0;db9ad1cb6cf9:34293 {}] regionserver.HRegionServer(2659): reportForDuty to master=db9ad1cb6cf9,38619,1730983259119 with port=34293, startcode=1730983260217 2024-11-07T12:41:02,146 DEBUG [master/db9ad1cb6cf9:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: db9ad1cb6cf9,38619,1730983259119 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-07T12:41:02,160 DEBUG [master/db9ad1cb6cf9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/db9ad1cb6cf9:0, corePoolSize=5, maxPoolSize=5 2024-11-07T12:41:02,160 DEBUG [master/db9ad1cb6cf9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/db9ad1cb6cf9:0, corePoolSize=5, maxPoolSize=5 2024-11-07T12:41:02,160 DEBUG [master/db9ad1cb6cf9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/db9ad1cb6cf9:0, corePoolSize=5, maxPoolSize=5 2024-11-07T12:41:02,160 DEBUG [master/db9ad1cb6cf9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/db9ad1cb6cf9:0, corePoolSize=5, maxPoolSize=5 2024-11-07T12:41:02,161 DEBUG [master/db9ad1cb6cf9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/db9ad1cb6cf9:0, corePoolSize=10, maxPoolSize=10 2024-11-07T12:41:02,161 DEBUG [master/db9ad1cb6cf9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/db9ad1cb6cf9:0, corePoolSize=1, maxPoolSize=1 2024-11-07T12:41:02,161 DEBUG [master/db9ad1cb6cf9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/db9ad1cb6cf9:0, corePoolSize=2, maxPoolSize=2 2024-11-07T12:41:02,162 DEBUG [master/db9ad1cb6cf9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/db9ad1cb6cf9:0, corePoolSize=1, maxPoolSize=1 2024-11-07T12:41:02,166 DEBUG [RS:0;db9ad1cb6cf9:34293 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-07T12:41:02,185 INFO [master/db9ad1cb6cf9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1730983292185 2024-11-07T12:41:02,188 INFO [master/db9ad1cb6cf9:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-07T12:41:02,189 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-07T12:41:02,190 INFO [master/db9ad1cb6cf9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-07T12:41:02,190 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-07T12:41:02,198 INFO [master/db9ad1cb6cf9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-07T12:41:02,198 INFO [master/db9ad1cb6cf9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-07T12:41:02,199 INFO [master/db9ad1cb6cf9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-07T12:41:02,199 INFO [master/db9ad1cb6cf9:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-07T12:41:02,204 INFO [master/db9ad1cb6cf9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-07T12:41:02,212 INFO [master/db9ad1cb6cf9:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-07T12:41:02,213 INFO [master/db9ad1cb6cf9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-07T12:41:02,213 INFO [master/db9ad1cb6cf9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-07T12:41:02,219 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:02,219 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-07T12:41:02,235 INFO [master/db9ad1cb6cf9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-07T12:41:02,241 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37679, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-07T12:41:02,247 INFO [master/db9ad1cb6cf9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-07T12:41:02,249 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38619 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:02,253 DEBUG [master/db9ad1cb6cf9:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/db9ad1cb6cf9:0:becomeActiveMaster-HFileCleaner.large.0-1730983262249,5,FailOnTimeoutGroup] 2024-11-07T12:41:02,257 DEBUG [master/db9ad1cb6cf9:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/db9ad1cb6cf9:0:becomeActiveMaster-HFileCleaner.small.0-1730983262253,5,FailOnTimeoutGroup] 2024-11-07T12:41:02,257 INFO [master/db9ad1cb6cf9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-07T12:41:02,257 INFO [master/db9ad1cb6cf9:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-07T12:41:02,259 INFO [master/db9ad1cb6cf9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-07T12:41:02,259 INFO [master/db9ad1cb6cf9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-07T12:41:02,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741831_1007 (size=1321) 2024-11-07T12:41:02,270 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-07T12:41:02,271 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4 2024-11-07T12:41:02,304 DEBUG [RS:0;db9ad1cb6cf9:34293 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-07T12:41:02,304 WARN [RS:0;db9ad1cb6cf9:34293 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-07T12:41:02,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741832_1008 (size=32) 2024-11-07T12:41:02,370 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:41:02,388 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-07T12:41:02,400 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-07T12:41:02,401 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:02,409 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T12:41:02,409 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-07T12:41:02,417 INFO [RS:0;db9ad1cb6cf9:34293 {}] regionserver.HRegionServer(2659): reportForDuty to master=db9ad1cb6cf9,38619,1730983259119 with port=34293, startcode=1730983260217 2024-11-07T12:41:02,420 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-07T12:41:02,420 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38619 {}] master.ServerManager(363): Checking decommissioned status of RegionServer db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:02,420 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:02,423 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38619 {}] master.ServerManager(517): Registering regionserver=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:02,441 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T12:41:02,441 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-07T12:41:02,446 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-07T12:41:02,447 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:02,449 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T12:41:02,449 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-07T12:41:02,454 DEBUG [RS:0;db9ad1cb6cf9:34293 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4 2024-11-07T12:41:02,455 DEBUG [RS:0;db9ad1cb6cf9:34293 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:37717 2024-11-07T12:41:02,455 DEBUG [RS:0;db9ad1cb6cf9:34293 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-07T12:41:02,455 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-07T12:41:02,455 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:02,460 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38619-0x1001a4271e00000, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-07T12:41:02,461 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T12:41:02,461 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-07T12:41:02,462 DEBUG [RS:0;db9ad1cb6cf9:34293 {}] zookeeper.ZKUtil(111): regionserver:34293-0x1001a4271e00001, quorum=127.0.0.1:49805, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:02,462 WARN [RS:0;db9ad1cb6cf9:34293 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-07T12:41:02,462 INFO [RS:0;db9ad1cb6cf9:34293 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-07T12:41:02,463 DEBUG [RS:0;db9ad1cb6cf9:34293 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/WALs/db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:02,469 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/hbase/meta/1588230740 2024-11-07T12:41:02,473 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/hbase/meta/1588230740 2024-11-07T12:41:02,475 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [db9ad1cb6cf9,34293,1730983260217] 2024-11-07T12:41:02,477 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-07T12:41:02,477 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-07T12:41:02,478 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-07T12:41:02,482 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-07T12:41:02,493 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T12:41:02,494 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71842411, jitterRate=0.07053534686565399}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-07T12:41:02,499 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1730983262371Initializing all the Stores at 1730983262374 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730983262374Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730983262385 (+11 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730983262385Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730983262385Cleaning up temporary data from old regions at 1730983262477 (+92 ms)Region opened successfully at 1730983262498 (+21 ms) 2024-11-07T12:41:02,499 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-07T12:41:02,499 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-07T12:41:02,499 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-07T12:41:02,499 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-07T12:41:02,499 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-07T12:41:02,501 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-07T12:41:02,501 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1730983262499Disabling compacts and flushes for region at 1730983262499Disabling writes for close at 1730983262499Writing region close event to WAL at 1730983262501 (+2 ms)Closed at 1730983262501 2024-11-07T12:41:02,511 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-07T12:41:02,511 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-07T12:41:02,524 INFO [RS:0;db9ad1cb6cf9:34293 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-07T12:41:02,524 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-07T12:41:02,536 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-07T12:41:02,541 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-07T12:41:02,557 INFO [RS:0;db9ad1cb6cf9:34293 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-07T12:41:02,584 INFO [RS:0;db9ad1cb6cf9:34293 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-07T12:41:02,584 INFO [RS:0;db9ad1cb6cf9:34293 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T12:41:02,593 INFO [RS:0;db9ad1cb6cf9:34293 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-07T12:41:02,599 INFO [RS:0;db9ad1cb6cf9:34293 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-07T12:41:02,601 INFO [RS:0;db9ad1cb6cf9:34293 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-07T12:41:02,601 DEBUG [RS:0;db9ad1cb6cf9:34293 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0, corePoolSize=1, maxPoolSize=1 2024-11-07T12:41:02,602 DEBUG [RS:0;db9ad1cb6cf9:34293 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/db9ad1cb6cf9:0, corePoolSize=1, maxPoolSize=1 2024-11-07T12:41:02,602 DEBUG [RS:0;db9ad1cb6cf9:34293 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/db9ad1cb6cf9:0, corePoolSize=1, maxPoolSize=1 2024-11-07T12:41:02,602 DEBUG [RS:0;db9ad1cb6cf9:34293 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0, corePoolSize=1, maxPoolSize=1 2024-11-07T12:41:02,602 DEBUG [RS:0;db9ad1cb6cf9:34293 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/db9ad1cb6cf9:0, corePoolSize=1, maxPoolSize=1 2024-11-07T12:41:02,602 DEBUG [RS:0;db9ad1cb6cf9:34293 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/db9ad1cb6cf9:0, corePoolSize=2, maxPoolSize=2 2024-11-07T12:41:02,602 DEBUG [RS:0;db9ad1cb6cf9:34293 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/db9ad1cb6cf9:0, corePoolSize=1, maxPoolSize=1 2024-11-07T12:41:02,603 DEBUG [RS:0;db9ad1cb6cf9:34293 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/db9ad1cb6cf9:0, corePoolSize=1, maxPoolSize=1 2024-11-07T12:41:02,603 DEBUG [RS:0;db9ad1cb6cf9:34293 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/db9ad1cb6cf9:0, corePoolSize=1, maxPoolSize=1 2024-11-07T12:41:02,603 DEBUG [RS:0;db9ad1cb6cf9:34293 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/db9ad1cb6cf9:0, corePoolSize=1, maxPoolSize=1 2024-11-07T12:41:02,603 DEBUG [RS:0;db9ad1cb6cf9:34293 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/db9ad1cb6cf9:0, corePoolSize=1, maxPoolSize=1 2024-11-07T12:41:02,604 DEBUG [RS:0;db9ad1cb6cf9:34293 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/db9ad1cb6cf9:0, corePoolSize=1, maxPoolSize=1 2024-11-07T12:41:02,604 DEBUG [RS:0;db9ad1cb6cf9:34293 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/db9ad1cb6cf9:0, corePoolSize=3, maxPoolSize=3 2024-11-07T12:41:02,604 DEBUG [RS:0;db9ad1cb6cf9:34293 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0, corePoolSize=3, maxPoolSize=3 2024-11-07T12:41:02,605 INFO [RS:0;db9ad1cb6cf9:34293 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-07T12:41:02,605 INFO [RS:0;db9ad1cb6cf9:34293 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-07T12:41:02,606 INFO [RS:0;db9ad1cb6cf9:34293 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T12:41:02,606 INFO [RS:0;db9ad1cb6cf9:34293 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-07T12:41:02,606 INFO [RS:0;db9ad1cb6cf9:34293 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-07T12:41:02,606 INFO [RS:0;db9ad1cb6cf9:34293 {}] hbase.ChoreService(168): Chore ScheduledChore name=db9ad1cb6cf9,34293,1730983260217-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-07T12:41:02,631 INFO [RS:0;db9ad1cb6cf9:34293 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-07T12:41:02,634 INFO [RS:0;db9ad1cb6cf9:34293 {}] hbase.ChoreService(168): Chore ScheduledChore name=db9ad1cb6cf9,34293,1730983260217-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T12:41:02,635 INFO [RS:0;db9ad1cb6cf9:34293 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T12:41:02,635 INFO [RS:0;db9ad1cb6cf9:34293 {}] regionserver.Replication(171): db9ad1cb6cf9,34293,1730983260217 started 2024-11-07T12:41:02,657 INFO [RS:0;db9ad1cb6cf9:34293 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T12:41:02,657 INFO [RS:0;db9ad1cb6cf9:34293 {}] regionserver.HRegionServer(1482): Serving as db9ad1cb6cf9,34293,1730983260217, RpcServer on db9ad1cb6cf9/172.17.0.2:34293, sessionid=0x1001a4271e00001 2024-11-07T12:41:02,658 DEBUG [RS:0;db9ad1cb6cf9:34293 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-07T12:41:02,658 DEBUG [RS:0;db9ad1cb6cf9:34293 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:02,659 DEBUG [RS:0;db9ad1cb6cf9:34293 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'db9ad1cb6cf9,34293,1730983260217' 2024-11-07T12:41:02,659 DEBUG [RS:0;db9ad1cb6cf9:34293 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-07T12:41:02,660 DEBUG [RS:0;db9ad1cb6cf9:34293 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-07T12:41:02,661 DEBUG [RS:0;db9ad1cb6cf9:34293 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-07T12:41:02,662 DEBUG [RS:0;db9ad1cb6cf9:34293 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-07T12:41:02,662 DEBUG [RS:0;db9ad1cb6cf9:34293 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:02,662 DEBUG [RS:0;db9ad1cb6cf9:34293 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'db9ad1cb6cf9,34293,1730983260217' 2024-11-07T12:41:02,662 DEBUG [RS:0;db9ad1cb6cf9:34293 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-07T12:41:02,663 DEBUG [RS:0;db9ad1cb6cf9:34293 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-07T12:41:02,663 DEBUG [RS:0;db9ad1cb6cf9:34293 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-07T12:41:02,664 INFO [RS:0;db9ad1cb6cf9:34293 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-07T12:41:02,664 INFO [RS:0;db9ad1cb6cf9:34293 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-07T12:41:02,695 WARN [db9ad1cb6cf9:38619 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-07T12:41:02,773 INFO [RS:0;db9ad1cb6cf9:34293 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-07T12:41:02,778 INFO [RS:0;db9ad1cb6cf9:34293 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=db9ad1cb6cf9%2C34293%2C1730983260217, suffix=, logDir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/WALs/db9ad1cb6cf9,34293,1730983260217, archiveDir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/oldWALs, maxLogs=32 2024-11-07T12:41:02,804 DEBUG [RS:0;db9ad1cb6cf9:34293 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/WALs/db9ad1cb6cf9,34293,1730983260217/db9ad1cb6cf9%2C34293%2C1730983260217.1730983262782, exclude list is [], retry=0 2024-11-07T12:41:02,811 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36063,DS-bd733cd2-30f1-4541-8f71-f5cc8e9227da,DISK] 2024-11-07T12:41:02,823 INFO [RS:0;db9ad1cb6cf9:34293 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/WALs/db9ad1cb6cf9,34293,1730983260217/db9ad1cb6cf9%2C34293%2C1730983260217.1730983262782 2024-11-07T12:41:02,824 DEBUG [RS:0;db9ad1cb6cf9:34293 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40023:40023)] 2024-11-07T12:41:02,949 DEBUG [db9ad1cb6cf9:38619 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-07T12:41:02,962 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:02,970 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as db9ad1cb6cf9,34293,1730983260217, state=OPENING 2024-11-07T12:41:02,976 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-07T12:41:02,980 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34293-0x1001a4271e00001, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T12:41:02,980 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38619-0x1001a4271e00000, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T12:41:02,983 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-07T12:41:02,984 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T12:41:02,984 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T12:41:02,985 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=db9ad1cb6cf9,34293,1730983260217}] 2024-11-07T12:41:03,179 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-07T12:41:03,186 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60077, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-07T12:41:03,213 INFO [RS_OPEN_META-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-07T12:41:03,214 INFO [RS_OPEN_META-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-07T12:41:03,215 INFO [RS_OPEN_META-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-07T12:41:03,227 INFO [RS_OPEN_META-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=db9ad1cb6cf9%2C34293%2C1730983260217.meta, suffix=.meta, logDir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/WALs/db9ad1cb6cf9,34293,1730983260217, archiveDir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/oldWALs, maxLogs=32 2024-11-07T12:41:03,265 DEBUG [RS_OPEN_META-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/WALs/db9ad1cb6cf9,34293,1730983260217/db9ad1cb6cf9%2C34293%2C1730983260217.meta.1730983263230.meta, exclude list is [], retry=0 2024-11-07T12:41:03,281 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36063,DS-bd733cd2-30f1-4541-8f71-f5cc8e9227da,DISK] 2024-11-07T12:41:03,300 INFO [RS_OPEN_META-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/WALs/db9ad1cb6cf9,34293,1730983260217/db9ad1cb6cf9%2C34293%2C1730983260217.meta.1730983263230.meta 2024-11-07T12:41:03,305 DEBUG [RS_OPEN_META-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40023:40023)] 2024-11-07T12:41:03,305 DEBUG [RS_OPEN_META-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-07T12:41:03,308 DEBUG [RS_OPEN_META-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-07T12:41:03,312 DEBUG [RS_OPEN_META-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-07T12:41:03,318 INFO [RS_OPEN_META-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-07T12:41:03,324 DEBUG [RS_OPEN_META-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-07T12:41:03,325 DEBUG [RS_OPEN_META-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:41:03,325 DEBUG [RS_OPEN_META-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-07T12:41:03,325 DEBUG [RS_OPEN_META-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-07T12:41:03,338 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-07T12:41:03,341 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-07T12:41:03,341 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:03,342 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T12:41:03,343 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-07T12:41:03,347 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-07T12:41:03,347 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:03,349 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T12:41:03,349 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-07T12:41:03,351 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-07T12:41:03,352 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:03,355 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T12:41:03,355 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-07T12:41:03,360 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-07T12:41:03,361 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:03,362 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T12:41:03,362 DEBUG [RS_OPEN_META-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-07T12:41:03,364 DEBUG [RS_OPEN_META-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/hbase/meta/1588230740 2024-11-07T12:41:03,389 DEBUG [RS_OPEN_META-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/hbase/meta/1588230740 2024-11-07T12:41:03,412 DEBUG [RS_OPEN_META-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-07T12:41:03,413 DEBUG [RS_OPEN_META-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-07T12:41:03,414 DEBUG [RS_OPEN_META-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-07T12:41:03,418 DEBUG [RS_OPEN_META-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-07T12:41:03,425 INFO [RS_OPEN_META-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66607744, jitterRate=-0.0074672698974609375}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-07T12:41:03,426 DEBUG [RS_OPEN_META-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-07T12:41:03,434 DEBUG [RS_OPEN_META-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1730983263326Writing region info on filesystem at 1730983263326Initializing all the Stores at 1730983263338 (+12 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730983263338Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730983263338Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730983263338Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730983263338Cleaning up temporary data from old regions at 1730983263413 (+75 ms)Running coprocessor post-open hooks at 1730983263426 (+13 ms)Region opened successfully at 1730983263434 (+8 ms) 2024-11-07T12:41:03,454 INFO [RS_OPEN_META-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1730983263161 2024-11-07T12:41:03,481 DEBUG [RS_OPEN_META-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-07T12:41:03,481 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:03,485 INFO [RS_OPEN_META-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-07T12:41:03,489 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as db9ad1cb6cf9,34293,1730983260217, state=OPEN 2024-11-07T12:41:03,503 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34293-0x1001a4271e00001, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T12:41:03,503 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T12:41:03,505 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38619-0x1001a4271e00000, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T12:41:03,506 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:03,508 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T12:41:03,535 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-07T12:41:03,535 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=db9ad1cb6cf9,34293,1730983260217 in 521 msec 2024-11-07T12:41:03,557 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-07T12:41:03,557 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 1.0130 sec 2024-11-07T12:41:03,560 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-07T12:41:03,560 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-07T12:41:03,596 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:41:03,598 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=-1] 2024-11-07T12:41:03,651 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:41:03,655 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34831, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:41:03,734 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.7120 sec 2024-11-07T12:41:03,734 INFO [master/db9ad1cb6cf9:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1730983263734, completionTime=-1 2024-11-07T12:41:03,738 INFO [master/db9ad1cb6cf9:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-07T12:41:03,739 DEBUG [master/db9ad1cb6cf9:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-07T12:41:03,823 INFO [master/db9ad1cb6cf9:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-07T12:41:03,823 INFO [master/db9ad1cb6cf9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1730983323823 2024-11-07T12:41:03,823 INFO [master/db9ad1cb6cf9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1730983383823 2024-11-07T12:41:03,823 INFO [master/db9ad1cb6cf9:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 84 msec 2024-11-07T12:41:03,826 INFO [master/db9ad1cb6cf9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=db9ad1cb6cf9,38619,1730983259119-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T12:41:03,827 INFO [master/db9ad1cb6cf9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=db9ad1cb6cf9,38619,1730983259119-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T12:41:03,827 INFO [master/db9ad1cb6cf9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=db9ad1cb6cf9,38619,1730983259119-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T12:41:03,835 INFO [master/db9ad1cb6cf9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-db9ad1cb6cf9:38619, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T12:41:03,851 INFO [master/db9ad1cb6cf9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-07T12:41:03,855 DEBUG [master/db9ad1cb6cf9:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-07T12:41:03,860 INFO [master/db9ad1cb6cf9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-07T12:41:03,915 INFO [master/db9ad1cb6cf9:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.498sec 2024-11-07T12:41:03,917 INFO [master/db9ad1cb6cf9:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-07T12:41:03,919 INFO [master/db9ad1cb6cf9:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-07T12:41:03,920 INFO [master/db9ad1cb6cf9:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-07T12:41:03,922 INFO [master/db9ad1cb6cf9:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-07T12:41:03,923 INFO [master/db9ad1cb6cf9:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-07T12:41:03,924 INFO [master/db9ad1cb6cf9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=db9ad1cb6cf9,38619,1730983259119-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-07T12:41:03,925 INFO [master/db9ad1cb6cf9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=db9ad1cb6cf9,38619,1730983259119-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-07T12:41:03,993 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a4346ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:41:03,996 DEBUG [master/db9ad1cb6cf9:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-07T12:41:03,997 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-07T12:41:03,997 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-07T12:41:03,997 INFO [master/db9ad1cb6cf9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-07T12:41:03,997 INFO [master/db9ad1cb6cf9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=db9ad1cb6cf9,38619,1730983259119-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T12:41:04,002 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request db9ad1cb6cf9,38619,-1 for getting cluster id 2024-11-07T12:41:04,007 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:41:04,038 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e542096-4b0e-4b01-9952-662501fc0f65' 2024-11-07T12:41:04,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:41:04,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e542096-4b0e-4b01-9952-662501fc0f65" 2024-11-07T12:41:04,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40f7a663, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:41:04,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [db9ad1cb6cf9,38619,-1] 2024-11-07T12:41:04,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:41:04,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:41:04,065 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36146, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:41:04,072 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bbb1cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:41:04,073 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:41:04,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=-1] 2024-11-07T12:41:04,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:41:04,091 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33308, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:41:04,152 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=db9ad1cb6cf9,38619,1730983259119 2024-11-07T12:41:04,187 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=209, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=1921, ProcessCount=11, AvailableMemoryMB=4399 2024-11-07T12:41:04,201 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-07T12:41:04,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is db9ad1cb6cf9,38619,1730983259119 2024-11-07T12:41:04,213 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3851ac6f 2024-11-07T12:41:04,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-07T12:41:04,219 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36148, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-07T12:41:04,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-07T12:41:04,228 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T12:41:04,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-07T12:41:04,253 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T12:41:04,256 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-07T12:41:04,256 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:04,261 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T12:41:04,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-07T12:41:04,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741835_1011 (size=1061) 2024-11-07T12:41:04,293 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4 2024-11-07T12:41:04,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741836_1012 (size=53) 2024-11-07T12:41:04,330 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:41:04,331 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 13f50260f7a437ef753d61fda36692fe, disabling compactions & flushes 2024-11-07T12:41:04,331 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:04,331 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:04,331 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. after waiting 0 ms 2024-11-07T12:41:04,331 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:04,331 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:04,331 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 13f50260f7a437ef753d61fda36692fe: Waiting for close lock at 1730983264330Disabling compacts and flushes for region at 1730983264330Disabling writes for close at 1730983264331 (+1 ms)Writing region close event to WAL at 1730983264331Closed at 1730983264331 2024-11-07T12:41:04,335 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T12:41:04,342 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1730983264335"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730983264335"}]},"ts":"1730983264335"} 2024-11-07T12:41:04,358 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-07T12:41:04,365 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T12:41:04,369 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730983264365"}]},"ts":"1730983264365"} 2024-11-07T12:41:04,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-07T12:41:04,389 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-07T12:41:04,391 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=13f50260f7a437ef753d61fda36692fe, ASSIGN}] 2024-11-07T12:41:04,403 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=13f50260f7a437ef753d61fda36692fe, ASSIGN 2024-11-07T12:41:04,419 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=13f50260f7a437ef753d61fda36692fe, ASSIGN; state=OFFLINE, location=db9ad1cb6cf9,34293,1730983260217; forceNewPlan=false, retain=false 2024-11-07T12:41:04,574 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=13f50260f7a437ef753d61fda36692fe, regionState=OPENING, regionLocation=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:04,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-07T12:41:04,597 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=13f50260f7a437ef753d61fda36692fe, ASSIGN because future has completed 2024-11-07T12:41:04,615 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217}] 2024-11-07T12:41:04,795 INFO [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:04,795 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} 2024-11-07T12:41:04,796 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 13f50260f7a437ef753d61fda36692fe 2024-11-07T12:41:04,796 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:41:04,796 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 13f50260f7a437ef753d61fda36692fe 2024-11-07T12:41:04,797 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 13f50260f7a437ef753d61fda36692fe 2024-11-07T12:41:04,800 INFO [StoreOpener-13f50260f7a437ef753d61fda36692fe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 13f50260f7a437ef753d61fda36692fe 2024-11-07T12:41:04,805 INFO [StoreOpener-13f50260f7a437ef753d61fda36692fe-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-07T12:41:04,805 INFO [StoreOpener-13f50260f7a437ef753d61fda36692fe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 13f50260f7a437ef753d61fda36692fe columnFamilyName A 2024-11-07T12:41:04,805 DEBUG [StoreOpener-13f50260f7a437ef753d61fda36692fe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:04,807 INFO [StoreOpener-13f50260f7a437ef753d61fda36692fe-1 {}] regionserver.HStore(327): Store=13f50260f7a437ef753d61fda36692fe/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:41:04,808 INFO [StoreOpener-13f50260f7a437ef753d61fda36692fe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 13f50260f7a437ef753d61fda36692fe 2024-11-07T12:41:04,813 INFO [StoreOpener-13f50260f7a437ef753d61fda36692fe-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-07T12:41:04,813 INFO [StoreOpener-13f50260f7a437ef753d61fda36692fe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 13f50260f7a437ef753d61fda36692fe columnFamilyName B 2024-11-07T12:41:04,814 DEBUG [StoreOpener-13f50260f7a437ef753d61fda36692fe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:04,815 INFO [StoreOpener-13f50260f7a437ef753d61fda36692fe-1 {}] regionserver.HStore(327): Store=13f50260f7a437ef753d61fda36692fe/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:41:04,815 INFO [StoreOpener-13f50260f7a437ef753d61fda36692fe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 13f50260f7a437ef753d61fda36692fe 2024-11-07T12:41:04,820 INFO [StoreOpener-13f50260f7a437ef753d61fda36692fe-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-07T12:41:04,821 INFO [StoreOpener-13f50260f7a437ef753d61fda36692fe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 13f50260f7a437ef753d61fda36692fe columnFamilyName C 2024-11-07T12:41:04,821 DEBUG [StoreOpener-13f50260f7a437ef753d61fda36692fe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:04,822 INFO [StoreOpener-13f50260f7a437ef753d61fda36692fe-1 {}] regionserver.HStore(327): Store=13f50260f7a437ef753d61fda36692fe/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:41:04,823 INFO [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:04,823 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 13f50260f7a437ef753d61fda36692fe 2024-11-07T12:41:04,825 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe 2024-11-07T12:41:04,826 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe 2024-11-07T12:41:04,828 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 13f50260f7a437ef753d61fda36692fe 2024-11-07T12:41:04,829 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 13f50260f7a437ef753d61fda36692fe 2024-11-07T12:41:04,829 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-07T12:41:04,832 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 13f50260f7a437ef753d61fda36692fe 2024-11-07T12:41:04,837 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T12:41:04,839 INFO [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 13f50260f7a437ef753d61fda36692fe; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65038440, jitterRate=-0.03085172176361084}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-07T12:41:04,839 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 13f50260f7a437ef753d61fda36692fe 2024-11-07T12:41:04,840 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 13f50260f7a437ef753d61fda36692fe: Running coprocessor pre-open hook at 1730983264797Writing region info on filesystem at 1730983264797Initializing all the Stores at 1730983264799 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730983264799Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730983264800 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730983264800Cleaning up temporary data from old regions at 1730983264829 (+29 ms)Running coprocessor post-open hooks at 1730983264839 (+10 ms)Region opened successfully at 1730983264840 (+1 ms) 2024-11-07T12:41:04,843 INFO [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., pid=6, masterSystemTime=1730983264785 2024-11-07T12:41:04,847 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:04,848 INFO [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:04,849 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=13f50260f7a437ef753d61fda36692fe, regionState=OPEN, openSeqNum=2, regionLocation=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:04,853 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 because future has completed 2024-11-07T12:41:04,861 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-07T12:41:04,861 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 in 240 msec 2024-11-07T12:41:04,869 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T12:41:04,869 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730983264869"}]},"ts":"1730983264869"} 2024-11-07T12:41:04,870 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-07T12:41:04,870 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=13f50260f7a437ef753d61fda36692fe, ASSIGN in 470 msec 2024-11-07T12:41:04,873 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-07T12:41:04,875 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T12:41:04,891 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 641 msec 2024-11-07T12:41:04,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-07T12:41:04,908 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:41:04,925 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad452c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:41:04,925 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request db9ad1cb6cf9,38619,-1 for getting cluster id 2024-11-07T12:41:04,925 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:41:04,930 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e542096-4b0e-4b01-9952-662501fc0f65' 2024-11-07T12:41:04,931 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:41:04,931 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e542096-4b0e-4b01-9952-662501fc0f65" 2024-11-07T12:41:04,931 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b12b609, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:41:04,932 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [db9ad1cb6cf9,38619,-1] 2024-11-07T12:41:04,932 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:41:04,933 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:41:04,936 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36158, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:41:04,939 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e008ec2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:41:04,939 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:41:04,942 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=-1] 2024-11-07T12:41:04,943 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:41:04,951 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33320, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:41:04,955 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-07T12:41:04,957 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is db9ad1cb6cf9,38619,1730983259119 2024-11-07T12:41:04,957 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@658d7f9b 2024-11-07T12:41:04,960 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-07T12:41:04,983 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36160, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-07T12:41:04,992 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19d419ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:41:04,992 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request db9ad1cb6cf9,38619,-1 for getting cluster id 2024-11-07T12:41:04,992 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:41:05,009 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e542096-4b0e-4b01-9952-662501fc0f65' 2024-11-07T12:41:05,010 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:41:05,010 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e542096-4b0e-4b01-9952-662501fc0f65" 2024-11-07T12:41:05,011 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44447930, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:41:05,011 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [db9ad1cb6cf9,38619,-1] 2024-11-07T12:41:05,012 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:41:05,021 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:41:05,026 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36182, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:41:05,033 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ac8d8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:41:05,037 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@130d4a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:41:05,038 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request db9ad1cb6cf9,38619,-1 for getting cluster id 2024-11-07T12:41:05,038 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:41:05,041 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e542096-4b0e-4b01-9952-662501fc0f65' 2024-11-07T12:41:05,048 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:41:05,049 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e542096-4b0e-4b01-9952-662501fc0f65" 2024-11-07T12:41:05,050 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22b8be13, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:41:05,050 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [db9ad1cb6cf9,38619,-1] 2024-11-07T12:41:05,050 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:41:05,061 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:41:05,065 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36196, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:41:05,067 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@797de42a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:41:05,069 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d35085, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:41:05,069 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request db9ad1cb6cf9,38619,-1 for getting cluster id 2024-11-07T12:41:05,069 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:41:05,085 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e542096-4b0e-4b01-9952-662501fc0f65' 2024-11-07T12:41:05,089 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:41:05,089 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e542096-4b0e-4b01-9952-662501fc0f65" 2024-11-07T12:41:05,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4112bfce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:41:05,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [db9ad1cb6cf9,38619,-1] 2024-11-07T12:41:05,091 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:41:05,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:41:05,097 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36226, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:41:05,104 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b2e42a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:41:05,107 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fb4b46b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:41:05,107 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request db9ad1cb6cf9,38619,-1 for getting cluster id 2024-11-07T12:41:05,107 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:41:05,147 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e542096-4b0e-4b01-9952-662501fc0f65' 2024-11-07T12:41:05,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:41:05,153 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e542096-4b0e-4b01-9952-662501fc0f65" 2024-11-07T12:41:05,153 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15b371cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:41:05,153 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [db9ad1cb6cf9,38619,-1] 2024-11-07T12:41:05,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:41:05,157 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:41:05,162 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36252, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:41:05,169 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bbb1025, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:41:05,174 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a1cce95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:41:05,174 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request db9ad1cb6cf9,38619,-1 for getting cluster id 2024-11-07T12:41:05,175 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:41:05,177 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e542096-4b0e-4b01-9952-662501fc0f65' 2024-11-07T12:41:05,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:41:05,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e542096-4b0e-4b01-9952-662501fc0f65" 2024-11-07T12:41:05,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39d3010f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:41:05,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [db9ad1cb6cf9,38619,-1] 2024-11-07T12:41:05,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:41:05,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:41:05,180 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36270, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:41:05,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64705bfa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:41:05,185 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ff89310, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:41:05,185 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request db9ad1cb6cf9,38619,-1 for getting cluster id 2024-11-07T12:41:05,185 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:41:05,186 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e542096-4b0e-4b01-9952-662501fc0f65' 2024-11-07T12:41:05,187 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:41:05,187 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e542096-4b0e-4b01-9952-662501fc0f65" 2024-11-07T12:41:05,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e154f6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:41:05,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [db9ad1cb6cf9,38619,-1] 2024-11-07T12:41:05,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:41:05,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:41:05,192 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36282, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:41:05,193 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17944247, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:41:05,197 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@761fbe54, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:41:05,197 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request db9ad1cb6cf9,38619,-1 for getting cluster id 2024-11-07T12:41:05,197 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:41:05,198 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e542096-4b0e-4b01-9952-662501fc0f65' 2024-11-07T12:41:05,198 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:41:05,198 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e542096-4b0e-4b01-9952-662501fc0f65" 2024-11-07T12:41:05,199 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13210d53, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:41:05,199 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [db9ad1cb6cf9,38619,-1] 2024-11-07T12:41:05,199 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:41:05,199 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:41:05,201 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36300, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:41:05,205 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3dda7125, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:41:05,209 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2019371a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:41:05,209 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request db9ad1cb6cf9,38619,-1 for getting cluster id 2024-11-07T12:41:05,210 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:41:05,212 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e542096-4b0e-4b01-9952-662501fc0f65' 2024-11-07T12:41:05,212 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:41:05,213 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e542096-4b0e-4b01-9952-662501fc0f65" 2024-11-07T12:41:05,213 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@caa0479, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:41:05,213 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [db9ad1cb6cf9,38619,-1] 2024-11-07T12:41:05,213 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:41:05,214 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:41:05,215 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36318, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:41:05,217 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8fc22e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:41:05,223 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36df70c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:41:05,223 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request db9ad1cb6cf9,38619,-1 for getting cluster id 2024-11-07T12:41:05,223 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:41:05,225 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e542096-4b0e-4b01-9952-662501fc0f65' 2024-11-07T12:41:05,226 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:41:05,226 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e542096-4b0e-4b01-9952-662501fc0f65" 2024-11-07T12:41:05,226 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@149eeca3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:41:05,226 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [db9ad1cb6cf9,38619,-1] 2024-11-07T12:41:05,227 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:41:05,227 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:41:05,229 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36342, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:41:05,230 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5734dc5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:41:05,242 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:41:05,243 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:41:05,244 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:41:05,244 DEBUG [Thread-145 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=-1] 2024-11-07T12:41:05,245 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=-1] 2024-11-07T12:41:05,245 DEBUG [Thread-145 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:41:05,245 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:41:05,246 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=-1] 2024-11-07T12:41:05,247 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:41:05,248 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33346, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:41:05,248 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33340, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:41:05,248 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33334, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:41:05,252 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:41:05,252 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:41:05,253 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:41:05,254 DEBUG [Thread-160 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=-1] 2024-11-07T12:41:05,254 DEBUG [Thread-160 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:41:05,255 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:41:05,256 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:41:05,256 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33350, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:41:05,260 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=-1] 2024-11-07T12:41:05,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=-1] 2024-11-07T12:41:05,261 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:41:05,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:41:05,262 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2] 2024-11-07T12:41:05,263 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33366, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:41:05,263 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33368, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:41:05,266 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=-1] 2024-11-07T12:41:05,266 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:41:05,261 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:41:05,270 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:41:05,270 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=-1] 2024-11-07T12:41:05,270 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:41:05,271 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2] 2024-11-07T12:41:05,271 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33370, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:41:05,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-07T12:41:05,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2] 2024-11-07T12:41:05,274 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33384, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:41:05,275 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=-1] 2024-11-07T12:41:05,275 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:41:05,275 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2] 2024-11-07T12:41:05,285 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2] 2024-11-07T12:41:05,285 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:41:05,290 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33394, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:41:05,290 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:41:05,292 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:41:05,293 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2] 2024-11-07T12:41:05,293 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2] 2024-11-07T12:41:05,306 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2] 2024-11-07T12:41:05,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2] 2024-11-07T12:41:05,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-07T12:41:05,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 13f50260f7a437ef753d61fda36692fe 2024-11-07T12:41:05,393 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 13f50260f7a437ef753d61fda36692fe 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-07T12:41:05,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13f50260f7a437ef753d61fda36692fe, store=A 2024-11-07T12:41:05,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:05,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13f50260f7a437ef753d61fda36692fe, store=B 2024-11-07T12:41:05,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:05,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13f50260f7a437ef753d61fda36692fe, store=C 2024-11-07T12:41:05,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:05,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-07T12:41:05,501 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-07T12:41:05,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:05,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. as already flushing 2024-11-07T12:41:05,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:05,524 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:05,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:05,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:05,628 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/887166ce52d64452878350f3f12ab91d is 50, key is test_row_0/A:col10/1730983265345/Put/seqid=0 2024-11-07T12:41:05,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-07T12:41:05,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:05,658 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1730983265551, max timestamp=1730983265620Num uniques 859; 2024-11-07T12:41:05,659 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1730983265551, max timestamp=1730983265606Num uniques 859; 2024-11-07T12:41:05,660 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-07T12:41:05,660 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-07T12:41:05,666 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1730983265551, max timestamp=1730983265620Num uniques 859; 2024-11-07T12:41:05,666 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-07T12:41:05,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33368 deadline: 1730983325592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:05,674 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:05,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:05,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:05,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741837_1013 (size=12001) 2024-11-07T12:41:05,728 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-07T12:41:05,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:05,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. as already flushing 2024-11-07T12:41:05,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:05,732 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:05,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:05,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:05,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:05,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33334 deadline: 1730983325774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:05,802 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:05,802 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:05,802 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:05,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:05,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33370 deadline: 1730983325778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:05,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:05,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:05,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:05,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:05,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:05,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33368 deadline: 1730983325790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:05,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:05,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:05,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:05,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33340 deadline: 1730983325789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:05,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:05,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33346 deadline: 1730983325795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:05,815 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:05,816 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:05,816 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:05,817 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:05,818 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:05,818 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:05,891 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-07T12:41:05,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:05,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. as already flushing 2024-11-07T12:41:05,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:05,905 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:05,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:05,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:05,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:05,925 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:05,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33370 deadline: 1730983325913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:05,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:05,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:05,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:05,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33334 deadline: 1730983325912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:05,934 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:05,935 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:05,935 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:05,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:05,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33346 deadline: 1730983325922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:05,938 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:05,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:05,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:05,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:05,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33340 deadline: 1730983325923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:05,940 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:05,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:05,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:05,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-07T12:41:06,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:06,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33368 deadline: 1730983326022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:06,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:06,029 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:06,029 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:06,061 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-07T12:41:06,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:06,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. as already flushing 2024-11-07T12:41:06,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:06,062 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:06,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:06,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:06,111 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/887166ce52d64452878350f3f12ab91d 2024-11-07T12:41:06,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:06,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33370 deadline: 1730983326134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:06,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:06,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:06,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:06,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:06,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33334 deadline: 1730983326144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:06,155 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:06,155 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:06,155 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:06,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:06,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33346 deadline: 1730983326144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:06,157 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:06,157 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:06,157 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:06,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:06,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33340 deadline: 1730983326157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:06,165 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:06,165 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:06,165 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:06,219 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-07T12:41:06,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:06,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. as already flushing 2024-11-07T12:41:06,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:06,220 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:06,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:06,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:06,330 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/debd4f8c1c994e9c9f32358e5cae6558 is 50, key is test_row_0/B:col10/1730983265345/Put/seqid=0 2024-11-07T12:41:06,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741838_1014 (size=12001) 2024-11-07T12:41:06,360 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/debd4f8c1c994e9c9f32358e5cae6558 2024-11-07T12:41:06,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:06,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33368 deadline: 1730983326344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:06,377 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-07T12:41:06,380 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:06,380 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:06,380 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:06,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:06,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. as already flushing 2024-11-07T12:41:06,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:06,381 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:06,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:06,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:06,470 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/9f2c21c4acf44a4b84aa28cbb43485dd is 50, key is test_row_0/C:col10/1730983265345/Put/seqid=0 2024-11-07T12:41:06,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-07T12:41:06,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:06,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33370 deadline: 1730983326464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:06,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:06,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:06,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:06,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:06,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33334 deadline: 1730983326464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:06,480 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:06,480 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:06,480 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:06,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:06,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33346 deadline: 1730983326464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:06,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:06,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:06,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:06,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:06,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33340 deadline: 1730983326473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:06,486 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:06,487 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:06,487 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:06,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741839_1015 (size=12001) 2024-11-07T12:41:06,511 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/9f2c21c4acf44a4b84aa28cbb43485dd 2024-11-07T12:41:06,527 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/887166ce52d64452878350f3f12ab91d as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/887166ce52d64452878350f3f12ab91d 2024-11-07T12:41:06,538 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-07T12:41:06,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:06,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. as already flushing 2024-11-07T12:41:06,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:06,539 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:06,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:06,539 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/887166ce52d64452878350f3f12ab91d, entries=150, sequenceid=12, filesize=11.7 K 2024-11-07T12:41:06,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:06,550 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/debd4f8c1c994e9c9f32358e5cae6558 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/debd4f8c1c994e9c9f32358e5cae6558 2024-11-07T12:41:06,600 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/debd4f8c1c994e9c9f32358e5cae6558, entries=150, sequenceid=12, filesize=11.7 K 2024-11-07T12:41:06,605 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/9f2c21c4acf44a4b84aa28cbb43485dd as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/9f2c21c4acf44a4b84aa28cbb43485dd 2024-11-07T12:41:06,627 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/9f2c21c4acf44a4b84aa28cbb43485dd, entries=150, sequenceid=12, filesize=11.7 K 2024-11-07T12:41:06,642 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 13f50260f7a437ef753d61fda36692fe in 1243ms, sequenceid=12, compaction requested=false 2024-11-07T12:41:06,644 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-07T12:41:06,646 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 13f50260f7a437ef753d61fda36692fe: 2024-11-07T12:41:06,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-07T12:41:06,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:06,706 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 13f50260f7a437ef753d61fda36692fe 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-11-07T12:41:06,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13f50260f7a437ef753d61fda36692fe, store=A 2024-11-07T12:41:06,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:41:06,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13f50260f7a437ef753d61fda36692fe, store=B 2024-11-07T12:41:06,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:41:06,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13f50260f7a437ef753d61fda36692fe, store=C 2024-11-07T12:41:06,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:41:06,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/8c70e7b9422f45088d9e8282cde78313 is 50, key is test_row_0/A:col10/1730983265768/Put/seqid=0 2024-11-07T12:41:06,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741840_1016 (size=12001) 2024-11-07T12:41:06,816 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/8c70e7b9422f45088d9e8282cde78313 2024-11-07T12:41:06,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/635e26c6edd74df4ac9b8674282b0522 is 50, key is test_row_0/B:col10/1730983265768/Put/seqid=0 2024-11-07T12:41:06,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 13f50260f7a437ef753d61fda36692fe 2024-11-07T12:41:06,901 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. as already flushing 2024-11-07T12:41:06,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741841_1017 (size=12001) 2024-11-07T12:41:07,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:07,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33368 deadline: 1730983327012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:07,068 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:07,068 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:07,068 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:07,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:07,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33370 deadline: 1730983327032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:07,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:07,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:07,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:07,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:07,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33334 deadline: 1730983327057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:07,125 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:07,125 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:07,125 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:07,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:07,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33346 deadline: 1730983327057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:07,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:07,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:07,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:07,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:07,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33340 deadline: 1730983327059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:07,132 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:07,133 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:07,133 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:07,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:07,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33368 deadline: 1730983327173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:07,207 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:07,207 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:07,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:07,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:07,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33370 deadline: 1730983327229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:07,269 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:07,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:07,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:07,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:07,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33346 deadline: 1730983327233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:07,271 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:07,271 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:07,271 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:07,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:07,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33334 deadline: 1730983327237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:07,274 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:07,274 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:07,274 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:07,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:07,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33340 deadline: 1730983327243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:07,277 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:07,277 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:07,277 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:07,314 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/635e26c6edd74df4ac9b8674282b0522 2024-11-07T12:41:07,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/f1bc8ccf47184853899915793b487071 is 50, key is test_row_0/C:col10/1730983265768/Put/seqid=0 2024-11-07T12:41:07,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:07,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33368 deadline: 1730983327413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:07,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:07,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:07,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:07,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741842_1018 (size=12001) 2024-11-07T12:41:07,479 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/f1bc8ccf47184853899915793b487071 2024-11-07T12:41:07,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-07T12:41:07,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/8c70e7b9422f45088d9e8282cde78313 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/8c70e7b9422f45088d9e8282cde78313 2024-11-07T12:41:07,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:07,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33346 deadline: 1730983327484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:07,504 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:07,504 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:07,504 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:07,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:07,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33370 deadline: 1730983327484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:07,505 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:07,505 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:07,505 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:07,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:07,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33334 deadline: 1730983327485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:07,513 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:07,513 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:07,513 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:07,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:07,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33340 deadline: 1730983327485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:07,515 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:07,515 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:07,515 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:07,516 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/8c70e7b9422f45088d9e8282cde78313, entries=150, sequenceid=46, filesize=11.7 K 2024-11-07T12:41:07,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/635e26c6edd74df4ac9b8674282b0522 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/635e26c6edd74df4ac9b8674282b0522 2024-11-07T12:41:07,579 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/635e26c6edd74df4ac9b8674282b0522, entries=150, sequenceid=46, filesize=11.7 K 2024-11-07T12:41:07,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/f1bc8ccf47184853899915793b487071 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/f1bc8ccf47184853899915793b487071 2024-11-07T12:41:07,628 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/f1bc8ccf47184853899915793b487071, entries=150, sequenceid=46, filesize=11.7 K 2024-11-07T12:41:07,632 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=60.38 KB/61830 for 13f50260f7a437ef753d61fda36692fe in 926ms, sequenceid=46, compaction requested=false 2024-11-07T12:41:07,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 13f50260f7a437ef753d61fda36692fe: 2024-11-07T12:41:07,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:07,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-07T12:41:07,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-07T12:41:07,641 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-07T12:41:07,641 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3440 sec 2024-11-07T12:41:07,645 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 2.3840 sec 2024-11-07T12:41:07,739 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 13f50260f7a437ef753d61fda36692fe 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-07T12:41:07,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13f50260f7a437ef753d61fda36692fe, store=A 2024-11-07T12:41:07,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:07,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13f50260f7a437ef753d61fda36692fe, store=B 2024-11-07T12:41:07,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:07,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13f50260f7a437ef753d61fda36692fe, store=C 2024-11-07T12:41:07,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:07,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 13f50260f7a437ef753d61fda36692fe 2024-11-07T12:41:07,754 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/984f6f600dd44af5a432476790bf6f1b is 50, key is test_row_0/A:col10/1730983267014/Put/seqid=0 2024-11-07T12:41:07,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741843_1019 (size=14341) 2024-11-07T12:41:07,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:07,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33370 deadline: 1730983327906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:07,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:07,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:07,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:07,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:07,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33368 deadline: 1730983327908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:07,940 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:07,940 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:07,940 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:07,949 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1730983267879, max timestamp=1730983267915Num uniques 859; 2024-11-07T12:41:07,951 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-07T12:41:07,955 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1730983267879, max timestamp=1730983267915Num uniques 859; 2024-11-07T12:41:07,956 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-07T12:41:07,993 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1730983267879, max timestamp=1730983267915Num uniques 859; 2024-11-07T12:41:07,993 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-07T12:41:08,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:08,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33346 deadline: 1730983328064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:08,079 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:08,079 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:08,079 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:08,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:08,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33340 deadline: 1730983328070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:08,110 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:08,110 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:08,110 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:08,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:08,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33334 deadline: 1730983328070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:08,113 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:08,113 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:08,113 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:08,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:08,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33368 deadline: 1730983328078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:08,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:08,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:08,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:08,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:08,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33370 deadline: 1730983328075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:08,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:08,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:08,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:08,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:08,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33346 deadline: 1730983328184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:08,223 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:08,223 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:08,223 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:08,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:08,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33340 deadline: 1730983328222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:08,226 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:08,226 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:08,226 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:08,233 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/984f6f600dd44af5a432476790bf6f1b 2024-11-07T12:41:08,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:08,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33334 deadline: 1730983328223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:08,246 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:08,248 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:08,248 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:08,264 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/f858673ce0cd4fee89de78ca69a0830f is 50, key is test_row_0/B:col10/1730983267014/Put/seqid=0 2024-11-07T12:41:08,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:08,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33368 deadline: 1730983328242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:08,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:08,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:08,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:08,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:08,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33370 deadline: 1730983328246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:08,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:08,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:08,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:08,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741844_1020 (size=12001) 2024-11-07T12:41:08,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:08,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33340 deadline: 1730983328433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:08,458 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:08,458 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:08,458 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:08,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:08,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33346 deadline: 1730983328433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:08,463 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:08,463 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:08,463 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:08,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:08,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33334 deadline: 1730983328452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:08,466 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:08,466 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:08,466 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:08,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:08,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33368 deadline: 1730983328472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:08,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:08,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33370 deadline: 1730983328482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:08,505 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:08,505 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:08,505 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:08,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:08,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:08,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:08,758 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/f858673ce0cd4fee89de78ca69a0830f 2024-11-07T12:41:08,789 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/8145a18ad4e0432c812ff4b68c2c71ae is 50, key is test_row_0/C:col10/1730983267014/Put/seqid=0 2024-11-07T12:41:08,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:08,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:08,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33334 deadline: 1730983328781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:08,804 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:08,804 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:08,804 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:08,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33346 deadline: 1730983328781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:08,805 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:08,805 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:08,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:08,805 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:08,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33340 deadline: 1730983328797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:08,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:08,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:08,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:08,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:08,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33370 deadline: 1730983328813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:08,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:08,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:08,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:08,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:08,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33368 deadline: 1730983328813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:08,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:08,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:08,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:08,866 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T12:41:08,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741845_1021 (size=12001) 2024-11-07T12:41:08,883 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/8145a18ad4e0432c812ff4b68c2c71ae 2024-11-07T12:41:08,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/984f6f600dd44af5a432476790bf6f1b as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/984f6f600dd44af5a432476790bf6f1b 2024-11-07T12:41:08,922 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/984f6f600dd44af5a432476790bf6f1b, entries=200, sequenceid=59, filesize=14.0 K 2024-11-07T12:41:08,926 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/f858673ce0cd4fee89de78ca69a0830f as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/f858673ce0cd4fee89de78ca69a0830f 2024-11-07T12:41:08,941 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/f858673ce0cd4fee89de78ca69a0830f, entries=150, sequenceid=59, filesize=11.7 K 2024-11-07T12:41:08,948 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/8145a18ad4e0432c812ff4b68c2c71ae as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/8145a18ad4e0432c812ff4b68c2c71ae 2024-11-07T12:41:08,966 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/8145a18ad4e0432c812ff4b68c2c71ae, entries=150, sequenceid=59, filesize=11.7 K 2024-11-07T12:41:08,970 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 13f50260f7a437ef753d61fda36692fe in 1231ms, sequenceid=59, compaction requested=true 2024-11-07T12:41:08,970 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 13f50260f7a437ef753d61fda36692fe: 2024-11-07T12:41:08,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13f50260f7a437ef753d61fda36692fe:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:41:08,994 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:41:08,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:41:08,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13f50260f7a437ef753d61fda36692fe:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:41:08,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:41:08,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13f50260f7a437ef753d61fda36692fe:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:41:08,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-07T12:41:08,996 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:41:09,007 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:41:09,009 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): 13f50260f7a437ef753d61fda36692fe/A is initiating minor compaction (all files) 2024-11-07T12:41:09,009 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13f50260f7a437ef753d61fda36692fe/A in TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:09,010 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/887166ce52d64452878350f3f12ab91d, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/8c70e7b9422f45088d9e8282cde78313, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/984f6f600dd44af5a432476790bf6f1b] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp, totalSize=37.4 K 2024-11-07T12:41:09,014 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:41:09,014 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 13f50260f7a437ef753d61fda36692fe/B is initiating minor compaction (all files) 2024-11-07T12:41:09,014 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13f50260f7a437ef753d61fda36692fe/B in TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:09,014 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/debd4f8c1c994e9c9f32358e5cae6558, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/635e26c6edd74df4ac9b8674282b0522, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/f858673ce0cd4fee89de78ca69a0830f] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp, totalSize=35.2 K 2024-11-07T12:41:09,016 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting debd4f8c1c994e9c9f32358e5cae6558, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1730983265274 2024-11-07T12:41:09,016 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 887166ce52d64452878350f3f12ab91d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1730983265274 2024-11-07T12:41:09,026 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c70e7b9422f45088d9e8282cde78313, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1730983265768 2024-11-07T12:41:09,026 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 635e26c6edd74df4ac9b8674282b0522, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1730983265768 2024-11-07T12:41:09,036 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 984f6f600dd44af5a432476790bf6f1b, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1730983267014 2024-11-07T12:41:09,038 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting f858673ce0cd4fee89de78ca69a0830f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1730983267014 2024-11-07T12:41:09,107 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-07T12:41:09,129 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13f50260f7a437ef753d61fda36692fe#A#compaction#10 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:41:09,130 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/e6b2c5e388124330a264171732c981e3 is 50, key is test_row_0/A:col10/1730983267014/Put/seqid=0 2024-11-07T12:41:09,132 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13f50260f7a437ef753d61fda36692fe#B#compaction#9 average throughput is 0.34 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:41:09,133 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/c910fe4263ab42e3a28160e585f63210 is 50, key is test_row_0/B:col10/1730983267014/Put/seqid=0 2024-11-07T12:41:09,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741846_1022 (size=12104) 2024-11-07T12:41:09,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741847_1023 (size=12104) 2024-11-07T12:41:09,286 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/e6b2c5e388124330a264171732c981e3 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/e6b2c5e388124330a264171732c981e3 2024-11-07T12:41:09,298 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/c910fe4263ab42e3a28160e585f63210 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/c910fe4263ab42e3a28160e585f63210 2024-11-07T12:41:09,328 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 13f50260f7a437ef753d61fda36692fe 3/3 column families, dataSize=120.76 KB heapSize=345.14 KB 2024-11-07T12:41:09,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13f50260f7a437ef753d61fda36692fe, store=A 2024-11-07T12:41:09,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:41:09,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13f50260f7a437ef753d61fda36692fe, store=B 2024-11-07T12:41:09,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:41:09,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13f50260f7a437ef753d61fda36692fe, store=C 2024-11-07T12:41:09,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:41:09,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 13f50260f7a437ef753d61fda36692fe 2024-11-07T12:41:09,363 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13f50260f7a437ef753d61fda36692fe/A of 13f50260f7a437ef753d61fda36692fe into e6b2c5e388124330a264171732c981e3(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:41:09,363 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13f50260f7a437ef753d61fda36692fe/B of 13f50260f7a437ef753d61fda36692fe into c910fe4263ab42e3a28160e585f63210(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:41:09,363 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13f50260f7a437ef753d61fda36692fe: 2024-11-07T12:41:09,363 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13f50260f7a437ef753d61fda36692fe: 2024-11-07T12:41:09,369 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., storeName=13f50260f7a437ef753d61fda36692fe/B, priority=13, startTime=1730983268996; duration=0sec 2024-11-07T12:41:09,370 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:41:09,371 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13f50260f7a437ef753d61fda36692fe:B 2024-11-07T12:41:09,371 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:41:09,373 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., storeName=13f50260f7a437ef753d61fda36692fe/A, priority=13, startTime=1730983268972; duration=0sec 2024-11-07T12:41:09,374 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:41:09,374 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13f50260f7a437ef753d61fda36692fe:A 2024-11-07T12:41:09,377 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/cac0449fd52048d49b229bf1e1cae684 is 50, key is test_row_0/A:col10/1730983268018/Put/seqid=0 2024-11-07T12:41:09,395 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:41:09,395 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 13f50260f7a437ef753d61fda36692fe/C is initiating minor compaction (all files) 2024-11-07T12:41:09,396 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13f50260f7a437ef753d61fda36692fe/C in TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:09,396 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/9f2c21c4acf44a4b84aa28cbb43485dd, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/f1bc8ccf47184853899915793b487071, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/8145a18ad4e0432c812ff4b68c2c71ae] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp, totalSize=35.2 K 2024-11-07T12:41:09,406 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f2c21c4acf44a4b84aa28cbb43485dd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1730983265274 2024-11-07T12:41:09,407 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting f1bc8ccf47184853899915793b487071, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1730983265768 2024-11-07T12:41:09,413 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 8145a18ad4e0432c812ff4b68c2c71ae, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1730983267014 2024-11-07T12:41:09,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741848_1024 (size=14341) 2024-11-07T12:41:09,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:09,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33370 deadline: 1730983329433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:09,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:09,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:09,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:09,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:09,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33340 deadline: 1730983329440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:09,489 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:09,489 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:09,489 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:09,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:09,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33346 deadline: 1730983329461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:09,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:09,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33334 deadline: 1730983329471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:09,493 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:09,493 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:09,493 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:09,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:09,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:09,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:09,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:09,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33368 deadline: 1730983329481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:09,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:09,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:09,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:09,509 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13f50260f7a437ef753d61fda36692fe#C#compaction#12 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:41:09,510 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/5ebf3ab4dd57492ca2baff06cb7efd4d is 50, key is test_row_0/C:col10/1730983267014/Put/seqid=0 2024-11-07T12:41:09,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-07T12:41:09,513 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:41:09,513 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:41:09,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-07T12:41:09,518 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:41:09,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-07T12:41:09,532 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:41:09,532 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:41:09,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741849_1025 (size=12104) 2024-11-07T12:41:09,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:09,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33370 deadline: 1730983329585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:09,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:09,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33340 deadline: 1730983329597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:09,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:09,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:09,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:09,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:09,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:09,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33334 deadline: 1730983329602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:09,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33346 deadline: 1730983329603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:09,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:09,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:09,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:09,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:09,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33368 deadline: 1730983329603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:09,624 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:09,629 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:09,629 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:09,631 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:09,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:09,631 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:09,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:09,631 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:09,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:09,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-07T12:41:09,689 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-07T12:41:09,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:09,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. as already flushing 2024-11-07T12:41:09,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:09,690 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:09,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:09,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:09,721 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-07T12:41:09,722 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-07T12:41:09,727 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-07T12:41:09,727 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-07T12:41:09,727 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-07T12:41:09,727 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-07T12:41:09,736 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-07T12:41:09,737 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-07T12:41:09,841 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/cac0449fd52048d49b229bf1e1cae684 2024-11-07T12:41:09,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:09,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33340 deadline: 1730983329836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:09,852 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:09,852 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:09,852 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:09,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-07T12:41:09,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:09,855 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-07T12:41:09,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33334 deadline: 1730983329842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:09,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:09,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. as already flushing 2024-11-07T12:41:09,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:09,856 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:09,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:09,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:09,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:09,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33370 deadline: 1730983329836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:09,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:09,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33368 deadline: 1730983329843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:09,862 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:09,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:09,862 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:09,862 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:09,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33346 deadline: 1730983329843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:09,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:09,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:09,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:09,866 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:09,866 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:09,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:09,866 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:09,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:09,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:09,871 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/18e5f240c667405fbfaf94be498c1e2e is 50, key is test_row_0/B:col10/1730983268018/Put/seqid=0 2024-11-07T12:41:09,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741850_1026 (size=12001) 2024-11-07T12:41:10,018 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/5ebf3ab4dd57492ca2baff06cb7efd4d as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/5ebf3ab4dd57492ca2baff06cb7efd4d 2024-11-07T12:41:10,021 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-07T12:41:10,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:10,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. as already flushing 2024-11-07T12:41:10,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:10,022 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:10,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:10,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:10,037 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13f50260f7a437ef753d61fda36692fe/C of 13f50260f7a437ef753d61fda36692fe into 5ebf3ab4dd57492ca2baff06cb7efd4d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:41:10,037 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13f50260f7a437ef753d61fda36692fe: 2024-11-07T12:41:10,037 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., storeName=13f50260f7a437ef753d61fda36692fe/C, priority=13, startTime=1730983268996; duration=0sec 2024-11-07T12:41:10,038 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:41:10,038 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13f50260f7a437ef753d61fda36692fe:C 2024-11-07T12:41:10,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-07T12:41:10,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:10,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33340 deadline: 1730983330165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:10,177 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:10,177 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:10,177 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:10,178 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-07T12:41:10,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:10,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. as already flushing 2024-11-07T12:41:10,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:10,179 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:10,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:10,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:10,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33370 deadline: 1730983330174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:10,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:10,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:10,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:10,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:10,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:10,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33368 deadline: 1730983330174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:10,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:10,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:10,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:10,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:10,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:10,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33334 deadline: 1730983330174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:10,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33346 deadline: 1730983330176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:10,184 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:10,184 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:10,184 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:10,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:10,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:10,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:10,342 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-07T12:41:10,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:10,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. as already flushing 2024-11-07T12:41:10,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:10,343 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:10,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:10,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:10,360 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/18e5f240c667405fbfaf94be498c1e2e 2024-11-07T12:41:10,425 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/99bcb8e4b7af48daa454930588fa90f9 is 50, key is test_row_0/C:col10/1730983268018/Put/seqid=0 2024-11-07T12:41:10,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741851_1027 (size=12001) 2024-11-07T12:41:10,492 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/99bcb8e4b7af48daa454930588fa90f9 2024-11-07T12:41:10,500 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-07T12:41:10,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:10,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. as already flushing 2024-11-07T12:41:10,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:10,501 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:10,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:10,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:10,517 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/cac0449fd52048d49b229bf1e1cae684 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/cac0449fd52048d49b229bf1e1cae684 2024-11-07T12:41:10,538 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/cac0449fd52048d49b229bf1e1cae684, entries=200, sequenceid=94, filesize=14.0 K 2024-11-07T12:41:10,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/18e5f240c667405fbfaf94be498c1e2e as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/18e5f240c667405fbfaf94be498c1e2e 2024-11-07T12:41:10,560 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/18e5f240c667405fbfaf94be498c1e2e, entries=150, sequenceid=94, filesize=11.7 K 2024-11-07T12:41:10,573 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/99bcb8e4b7af48daa454930588fa90f9 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/99bcb8e4b7af48daa454930588fa90f9 2024-11-07T12:41:10,605 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/99bcb8e4b7af48daa454930588fa90f9, entries=150, sequenceid=94, filesize=11.7 K 2024-11-07T12:41:10,607 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~345.09 KB/353376, currentSize=60.38 KB/61830 for 13f50260f7a437ef753d61fda36692fe in 1279ms, sequenceid=94, compaction requested=false 2024-11-07T12:41:10,608 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 13f50260f7a437ef753d61fda36692fe: 2024-11-07T12:41:10,665 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-07T12:41:10,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:10,666 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 13f50260f7a437ef753d61fda36692fe 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-07T12:41:10,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13f50260f7a437ef753d61fda36692fe, store=A 2024-11-07T12:41:10,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:10,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13f50260f7a437ef753d61fda36692fe, store=B 2024-11-07T12:41:10,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:10,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13f50260f7a437ef753d61fda36692fe, store=C 2024-11-07T12:41:10,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:10,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-07T12:41:10,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/7ce98257f13c41c5841bedffb34c4ef5 is 50, key is test_row_0/A:col10/1730983269460/Put/seqid=0 2024-11-07T12:41:10,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 13f50260f7a437ef753d61fda36692fe 2024-11-07T12:41:10,686 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. as already flushing 2024-11-07T12:41:10,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741852_1028 (size=12001) 2024-11-07T12:41:10,736 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/7ce98257f13c41c5841bedffb34c4ef5 2024-11-07T12:41:10,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/67124a5d88fe4e69857c3d161412b606 is 50, key is test_row_0/B:col10/1730983269460/Put/seqid=0 2024-11-07T12:41:10,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:10,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33334 deadline: 1730983330780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:10,800 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:10,800 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:10,801 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:10,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:10,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33368 deadline: 1730983330785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:10,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:10,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:10,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:10,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:10,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33346 deadline: 1730983330787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:10,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:10,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:10,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:10,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:10,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33370 deadline: 1730983330790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:10,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:10,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:10,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:10,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:10,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33340 deadline: 1730983330800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:10,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:10,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:10,832 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:10,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741853_1029 (size=12001) 2024-11-07T12:41:10,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:10,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33368 deadline: 1730983330913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:10,927 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:10,927 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:10,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:10,927 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:10,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33346 deadline: 1730983330913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:10,928 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:10,928 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:10,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:10,928 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:10,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33334 deadline: 1730983330914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:10,929 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:10,929 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:10,929 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:10,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:10,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33370 deadline: 1730983330932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:10,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:10,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:10,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:10,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:10,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33340 deadline: 1730983330942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:10,950 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:10,950 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:10,950 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:11,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:11,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33368 deadline: 1730983331132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:11,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:11,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:11,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:11,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:11,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33346 deadline: 1730983331133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:11,141 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:11,141 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:11,141 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:11,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:11,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33370 deadline: 1730983331142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:11,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:11,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:11,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33334 deadline: 1730983331142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:11,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:11,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:11,149 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:11,149 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:11,149 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:11,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:11,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33340 deadline: 1730983331161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:11,167 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:11,167 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:11,167 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:11,238 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/67124a5d88fe4e69857c3d161412b606 2024-11-07T12:41:11,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/45523d94041e41c09e3d2554757ee7d8 is 50, key is test_row_0/C:col10/1730983269460/Put/seqid=0 2024-11-07T12:41:11,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741854_1030 (size=12001) 2024-11-07T12:41:11,348 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/45523d94041e41c09e3d2554757ee7d8 2024-11-07T12:41:11,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/7ce98257f13c41c5841bedffb34c4ef5 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/7ce98257f13c41c5841bedffb34c4ef5 2024-11-07T12:41:11,420 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/7ce98257f13c41c5841bedffb34c4ef5, entries=150, sequenceid=109, filesize=11.7 K 2024-11-07T12:41:11,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/67124a5d88fe4e69857c3d161412b606 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/67124a5d88fe4e69857c3d161412b606 2024-11-07T12:41:11,455 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/67124a5d88fe4e69857c3d161412b606, entries=150, sequenceid=109, filesize=11.7 K 2024-11-07T12:41:11,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/45523d94041e41c09e3d2554757ee7d8 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/45523d94041e41c09e3d2554757ee7d8 2024-11-07T12:41:11,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:11,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33368 deadline: 1730983331445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:11,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:11,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33370 deadline: 1730983331454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:11,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:11,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:11,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33346 deadline: 1730983331454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:11,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:11,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:11,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:11,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:11,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:11,477 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:11,477 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:11,478 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:11,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:11,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33334 deadline: 1730983331455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:11,479 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:11,479 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:11,479 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:11,485 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/45523d94041e41c09e3d2554757ee7d8, entries=150, sequenceid=109, filesize=11.7 K 2024-11-07T12:41:11,496 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 13f50260f7a437ef753d61fda36692fe in 830ms, sequenceid=109, compaction requested=true 2024-11-07T12:41:11,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 13f50260f7a437ef753d61fda36692fe: 2024-11-07T12:41:11,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:11,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-07T12:41:11,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-07T12:41:11,506 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-07T12:41:11,506 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9700 sec 2024-11-07T12:41:11,510 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.9940 sec 2024-11-07T12:41:11,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 13f50260f7a437ef753d61fda36692fe 2024-11-07T12:41:11,515 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1730983270777, max timestamp=1730983271487Num uniques 859; 2024-11-07T12:41:11,515 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-07T12:41:11,517 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 13f50260f7a437ef753d61fda36692fe 3/3 column families, dataSize=89.45 KB heapSize=262.11 KB 2024-11-07T12:41:11,517 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13f50260f7a437ef753d61fda36692fe, store=A 2024-11-07T12:41:11,520 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1730983270777, max timestamp=1730983271487Num uniques 859; 2024-11-07T12:41:11,520 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-07T12:41:11,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:41:11,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13f50260f7a437ef753d61fda36692fe, store=B 2024-11-07T12:41:11,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:41:11,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13f50260f7a437ef753d61fda36692fe, store=C 2024-11-07T12:41:11,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:41:11,539 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/6a2ec1ae72ca4cbfb08375698bb9e5e8 is 50, key is test_row_0/A:col10/1730983271487/Put/seqid=0 2024-11-07T12:41:11,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741855_1031 (size=14391) 2024-11-07T12:41:11,595 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/6a2ec1ae72ca4cbfb08375698bb9e5e8 2024-11-07T12:41:11,624 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/5a37914d7b9d4902baee54a711860dd3 is 50, key is test_row_0/B:col10/1730983271487/Put/seqid=0 2024-11-07T12:41:11,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741856_1032 (size=12051) 2024-11-07T12:41:11,656 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/5a37914d7b9d4902baee54a711860dd3 2024-11-07T12:41:11,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-07T12:41:11,683 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:41:11,684 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:41:11,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-07T12:41:11,692 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/c2c96961925f423aa6f45edfa973167f is 50, key is test_row_0/C:col10/1730983271487/Put/seqid=0 2024-11-07T12:41:11,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-07T12:41:11,695 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:41:11,697 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:41:11,697 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:41:11,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741857_1033 (size=12051) 2024-11-07T12:41:11,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-07T12:41:11,852 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-07T12:41:11,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:11,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. as already flushing 2024-11-07T12:41:11,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:11,853 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:11,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:11,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:11,903 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1730983271752, max timestamp=1730983271890Num uniques 859; 2024-11-07T12:41:11,904 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-07T12:41:11,906 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1730983271752, max timestamp=1730983271890Num uniques 859; 2024-11-07T12:41:11,907 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-07T12:41:12,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-07T12:41:12,017 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-07T12:41:12,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:12,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. as already flushing 2024-11-07T12:41:12,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:12,017 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:12,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:12,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:12,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:12,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33346 deadline: 1730983332023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:12,043 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:12,043 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:12,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:12,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:12,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33368 deadline: 1730983332025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:12,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:12,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33370 deadline: 1730983332025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:12,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:12,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:12,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:12,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:12,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:12,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:12,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:12,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33334 deadline: 1730983332036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:12,061 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:12,061 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:12,061 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:12,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:12,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33340 deadline: 1730983332044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:12,065 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:12,065 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:12,065 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:12,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:12,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33346 deadline: 1730983332152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:12,162 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/c2c96961925f423aa6f45edfa973167f 2024-11-07T12:41:12,163 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:12,163 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:12,163 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:12,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:12,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33368 deadline: 1730983332162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:12,169 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:12,169 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:12,169 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:12,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:12,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33370 deadline: 1730983332163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:12,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:12,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:12,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:12,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:12,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33340 deadline: 1730983332173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:12,192 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:12,192 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:12,192 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:12,193 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-07T12:41:12,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:12,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:12,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. as already flushing 2024-11-07T12:41:12,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33334 deadline: 1730983332176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:12,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:12,194 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:12,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:12,195 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:12,195 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:12,195 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:12,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:12,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/6a2ec1ae72ca4cbfb08375698bb9e5e8 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/6a2ec1ae72ca4cbfb08375698bb9e5e8 2024-11-07T12:41:12,249 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/6a2ec1ae72ca4cbfb08375698bb9e5e8, entries=200, sequenceid=130, filesize=14.1 K 2024-11-07T12:41:12,264 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/5a37914d7b9d4902baee54a711860dd3 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/5a37914d7b9d4902baee54a711860dd3 2024-11-07T12:41:12,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-07T12:41:12,326 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/5a37914d7b9d4902baee54a711860dd3, entries=150, sequenceid=130, filesize=11.8 K 2024-11-07T12:41:12,330 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/c2c96961925f423aa6f45edfa973167f as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/c2c96961925f423aa6f45edfa973167f 2024-11-07T12:41:12,351 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-07T12:41:12,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:12,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. as already flushing 2024-11-07T12:41:12,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:12,352 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:12,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:12,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:12,372 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/c2c96961925f423aa6f45edfa973167f, entries=150, sequenceid=130, filesize=11.8 K 2024-11-07T12:41:12,374 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~58.14 KB/59540, heapSize ~164.13 KB/168064, currentSize=123.00 KB/125950 for 13f50260f7a437ef753d61fda36692fe in 858ms, sequenceid=130, compaction requested=true 2024-11-07T12:41:12,375 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 13f50260f7a437ef753d61fda36692fe: 2024-11-07T12:41:12,375 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:41:12,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13f50260f7a437ef753d61fda36692fe:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:41:12,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:41:12,378 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:41:12,379 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52837 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:41:12,379 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): 13f50260f7a437ef753d61fda36692fe/A is initiating minor compaction (all files) 2024-11-07T12:41:12,379 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13f50260f7a437ef753d61fda36692fe/A in TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:12,379 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/e6b2c5e388124330a264171732c981e3, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/cac0449fd52048d49b229bf1e1cae684, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/7ce98257f13c41c5841bedffb34c4ef5, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/6a2ec1ae72ca4cbfb08375698bb9e5e8] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp, totalSize=51.6 K 2024-11-07T12:41:12,380 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6b2c5e388124330a264171732c981e3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1730983267014 2024-11-07T12:41:12,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13f50260f7a437ef753d61fda36692fe:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:41:12,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:41:12,381 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48157 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:41:12,382 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 13f50260f7a437ef753d61fda36692fe/B is initiating minor compaction (all files) 2024-11-07T12:41:12,382 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13f50260f7a437ef753d61fda36692fe/B in TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:12,382 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/c910fe4263ab42e3a28160e585f63210, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/18e5f240c667405fbfaf94be498c1e2e, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/67124a5d88fe4e69857c3d161412b606, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/5a37914d7b9d4902baee54a711860dd3] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp, totalSize=47.0 K 2024-11-07T12:41:12,382 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting cac0449fd52048d49b229bf1e1cae684, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1730983268018 2024-11-07T12:41:12,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 13f50260f7a437ef753d61fda36692fe 2024-11-07T12:41:12,389 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting c910fe4263ab42e3a28160e585f63210, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1730983267014 2024-11-07T12:41:12,390 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ce98257f13c41c5841bedffb34c4ef5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1730983269370 2024-11-07T12:41:12,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13f50260f7a437ef753d61fda36692fe:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:41:12,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:41:12,391 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 18e5f240c667405fbfaf94be498c1e2e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1730983268018 2024-11-07T12:41:12,391 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a2ec1ae72ca4cbfb08375698bb9e5e8, keycount=200, bloomtype=ROW, size=14.1 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1730983270777 2024-11-07T12:41:12,397 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 67124a5d88fe4e69857c3d161412b606, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1730983269370 2024-11-07T12:41:12,398 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 13f50260f7a437ef753d61fda36692fe 3/3 column families, dataSize=136.42 KB heapSize=393.47 KB 2024-11-07T12:41:12,398 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a37914d7b9d4902baee54a711860dd3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1730983270777 2024-11-07T12:41:12,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13f50260f7a437ef753d61fda36692fe, store=A 2024-11-07T12:41:12,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:41:12,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13f50260f7a437ef753d61fda36692fe, store=B 2024-11-07T12:41:12,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:41:12,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13f50260f7a437ef753d61fda36692fe, store=C 2024-11-07T12:41:12,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:41:12,455 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/3ffc455d920d4467a2c74ed7a9e9da38 is 50, key is test_row_0/A:col10/1730983271964/Put/seqid=0 2024-11-07T12:41:12,463 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13f50260f7a437ef753d61fda36692fe#A#compaction#22 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:41:12,464 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/68784e5391194706a714e051045ee786 is 50, key is test_row_0/A:col10/1730983271487/Put/seqid=0 2024-11-07T12:41:12,469 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13f50260f7a437ef753d61fda36692fe#B#compaction#23 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:41:12,470 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/2ab0eb7528cd40a0b1ad4741e4353362 is 50, key is test_row_0/B:col10/1730983271487/Put/seqid=0 2024-11-07T12:41:12,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:12,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33346 deadline: 1730983332449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:12,505 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:12,505 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:12,505 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:12,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:12,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33340 deadline: 1730983332458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:12,506 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:12,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33334 deadline: 1730983332464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:12,508 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:12,508 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:12,508 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:12,509 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:12,509 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:12,509 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:12,511 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-07T12:41:12,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:12,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. as already flushing 2024-11-07T12:41:12,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:12,512 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:12,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:12,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:12,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741858_1034 (size=16931) 2024-11-07T12:41:12,537 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/3ffc455d920d4467a2c74ed7a9e9da38 2024-11-07T12:41:12,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741859_1035 (size=12291) 2024-11-07T12:41:12,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741860_1036 (size=12291) 2024-11-07T12:41:12,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:12,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33370 deadline: 1730983332504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:12,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:12,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33368 deadline: 1730983332515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:12,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:12,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:12,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:12,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:12,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:12,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:12,585 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/e78b648d4c224c56b73e4ee612eb63af is 50, key is test_row_0/B:col10/1730983271964/Put/seqid=0 2024-11-07T12:41:12,593 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/2ab0eb7528cd40a0b1ad4741e4353362 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/2ab0eb7528cd40a0b1ad4741e4353362 2024-11-07T12:41:12,621 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 13f50260f7a437ef753d61fda36692fe/B of 13f50260f7a437ef753d61fda36692fe into 2ab0eb7528cd40a0b1ad4741e4353362(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:41:12,624 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13f50260f7a437ef753d61fda36692fe: 2024-11-07T12:41:12,624 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., storeName=13f50260f7a437ef753d61fda36692fe/B, priority=12, startTime=1730983272377; duration=0sec 2024-11-07T12:41:12,625 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:41:12,625 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13f50260f7a437ef753d61fda36692fe:B 2024-11-07T12:41:12,625 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:41:12,643 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48157 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:41:12,643 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 13f50260f7a437ef753d61fda36692fe/C is initiating minor compaction (all files) 2024-11-07T12:41:12,643 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13f50260f7a437ef753d61fda36692fe/C in TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:12,643 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/5ebf3ab4dd57492ca2baff06cb7efd4d, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/99bcb8e4b7af48daa454930588fa90f9, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/45523d94041e41c09e3d2554757ee7d8, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/c2c96961925f423aa6f45edfa973167f] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp, totalSize=47.0 K 2024-11-07T12:41:12,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:12,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33346 deadline: 1730983332613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:12,645 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ebf3ab4dd57492ca2baff06cb7efd4d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1730983267014 2024-11-07T12:41:12,649 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 99bcb8e4b7af48daa454930588fa90f9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1730983268018 2024-11-07T12:41:12,649 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 45523d94041e41c09e3d2554757ee7d8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1730983269370 2024-11-07T12:41:12,652 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting c2c96961925f423aa6f45edfa973167f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1730983270777 2024-11-07T12:41:12,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741861_1037 (size=12151) 2024-11-07T12:41:12,659 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:12,659 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/e78b648d4c224c56b73e4ee612eb63af 2024-11-07T12:41:12,659 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:12,659 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:12,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:12,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33334 deadline: 1730983332614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:12,660 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:12,660 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:12,660 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:12,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:12,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33340 deadline: 1730983332614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:12,661 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:12,661 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:12,661 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:12,681 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-07T12:41:12,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:12,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. as already flushing 2024-11-07T12:41:12,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:12,682 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:12,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:12,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:12,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:12,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33370 deadline: 1730983332680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:12,715 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/310e2a7ec25c41b99df1f6e06f8dff7c is 50, key is test_row_0/C:col10/1730983271964/Put/seqid=0 2024-11-07T12:41:12,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:12,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33368 deadline: 1730983332680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:12,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:12,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:12,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:12,724 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:12,724 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:12,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:12,736 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13f50260f7a437ef753d61fda36692fe#C#compaction#26 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:41:12,737 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/0f457255476e43daa9c9065a2bef4730 is 50, key is test_row_0/C:col10/1730983271487/Put/seqid=0 2024-11-07T12:41:12,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741862_1038 (size=12151) 2024-11-07T12:41:12,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-07T12:41:12,843 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-07T12:41:12,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:12,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. as already flushing 2024-11-07T12:41:12,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:12,844 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:12,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:12,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:12,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741863_1039 (size=12291) 2024-11-07T12:41:12,870 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/0f457255476e43daa9c9065a2bef4730 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/0f457255476e43daa9c9065a2bef4730 2024-11-07T12:41:12,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:12,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33346 deadline: 1730983332867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:12,872 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:12,872 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:12,872 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:12,884 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 13f50260f7a437ef753d61fda36692fe/C of 13f50260f7a437ef753d61fda36692fe into 0f457255476e43daa9c9065a2bef4730(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:41:12,886 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13f50260f7a437ef753d61fda36692fe: 2024-11-07T12:41:12,886 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., storeName=13f50260f7a437ef753d61fda36692fe/C, priority=12, startTime=1730983272391; duration=0sec 2024-11-07T12:41:12,887 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:41:12,887 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13f50260f7a437ef753d61fda36692fe:C 2024-11-07T12:41:12,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:12,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33340 deadline: 1730983332872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:12,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:12,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:12,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:12,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:12,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33334 deadline: 1730983332872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:12,891 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:12,892 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:12,892 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:12,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:12,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33370 deadline: 1730983332925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:12,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:12,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:12,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:12,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:12,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33368 deadline: 1730983332935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:12,947 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:12,947 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:12,947 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:12,976 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/68784e5391194706a714e051045ee786 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/68784e5391194706a714e051045ee786 2024-11-07T12:41:12,999 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 13f50260f7a437ef753d61fda36692fe/A of 13f50260f7a437ef753d61fda36692fe into 68784e5391194706a714e051045ee786(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:41:12,999 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13f50260f7a437ef753d61fda36692fe: 2024-11-07T12:41:12,999 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., storeName=13f50260f7a437ef753d61fda36692fe/A, priority=12, startTime=1730983272375; duration=0sec 2024-11-07T12:41:13,000 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:41:13,000 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13f50260f7a437ef753d61fda36692fe:A 2024-11-07T12:41:13,000 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-07T12:41:13,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:13,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. as already flushing 2024-11-07T12:41:13,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:13,001 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:13,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:13,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:13,161 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-07T12:41:13,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:13,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. as already flushing 2024-11-07T12:41:13,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:13,165 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:13,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:13,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:13,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:13,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33346 deadline: 1730983333181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:13,183 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:13,183 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:13,183 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:13,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:13,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33334 deadline: 1730983333202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:13,207 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:13,207 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:13,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:13,207 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:13,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33340 deadline: 1730983333203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:13,208 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:13,208 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:13,208 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:13,232 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/310e2a7ec25c41b99df1f6e06f8dff7c 2024-11-07T12:41:13,238 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:13,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33370 deadline: 1730983333232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:13,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:13,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:13,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:13,252 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/3ffc455d920d4467a2c74ed7a9e9da38 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/3ffc455d920d4467a2c74ed7a9e9da38 2024-11-07T12:41:13,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:13,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33368 deadline: 1730983333252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:13,261 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:13,261 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:13,261 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:13,277 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/3ffc455d920d4467a2c74ed7a9e9da38, entries=250, sequenceid=162, filesize=16.5 K 2024-11-07T12:41:13,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/e78b648d4c224c56b73e4ee612eb63af as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/e78b648d4c224c56b73e4ee612eb63af 2024-11-07T12:41:13,329 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/e78b648d4c224c56b73e4ee612eb63af, entries=150, sequenceid=162, filesize=11.9 K 2024-11-07T12:41:13,332 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-07T12:41:13,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:13,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. as already flushing 2024-11-07T12:41:13,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:13,333 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:13,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:13,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:13,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/310e2a7ec25c41b99df1f6e06f8dff7c as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/310e2a7ec25c41b99df1f6e06f8dff7c 2024-11-07T12:41:13,353 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/310e2a7ec25c41b99df1f6e06f8dff7c, entries=150, sequenceid=162, filesize=11.9 K 2024-11-07T12:41:13,355 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~136.42 KB/139690, heapSize ~393.42 KB/402864, currentSize=46.96 KB/48090 for 13f50260f7a437ef753d61fda36692fe in 957ms, sequenceid=162, compaction requested=false 2024-11-07T12:41:13,355 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 13f50260f7a437ef753d61fda36692fe: 2024-11-07T12:41:13,488 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-07T12:41:13,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:13,489 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 13f50260f7a437ef753d61fda36692fe 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-07T12:41:13,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13f50260f7a437ef753d61fda36692fe, store=A 2024-11-07T12:41:13,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:13,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13f50260f7a437ef753d61fda36692fe, store=B 2024-11-07T12:41:13,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:13,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13f50260f7a437ef753d61fda36692fe, store=C 2024-11-07T12:41:13,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:13,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/4d7e0508c5014a4a8c167717b99a6336 is 50, key is test_row_0/A:col10/1730983272487/Put/seqid=0 2024-11-07T12:41:13,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741864_1040 (size=12151) 2024-11-07T12:41:13,548 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/4d7e0508c5014a4a8c167717b99a6336 2024-11-07T12:41:13,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/3e17a8db69e04f15bf88f3720ea288e9 is 50, key is test_row_0/B:col10/1730983272487/Put/seqid=0 2024-11-07T12:41:13,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741865_1041 (size=12151) 2024-11-07T12:41:13,704 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. as already flushing 2024-11-07T12:41:13,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 13f50260f7a437ef753d61fda36692fe 2024-11-07T12:41:13,799 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1730983273749, max timestamp=1730983273796Num uniques 859; 2024-11-07T12:41:13,799 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1730983273749, max timestamp=1730983273796Num uniques 859; 2024-11-07T12:41:13,799 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-07T12:41:13,799 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-07T12:41:13,802 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1730983273749, max timestamp=1730983273796Num uniques 859; 2024-11-07T12:41:13,802 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-07T12:41:13,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-07T12:41:13,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:13,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33370 deadline: 1730983333840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:13,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:13,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:13,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:13,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:13,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33334 deadline: 1730983333854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:13,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:13,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33346 deadline: 1730983333854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:13,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:13,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:13,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33368 deadline: 1730983333855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:13,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:13,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:13,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:13,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:13,869 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:13,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:13,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33340 deadline: 1730983333855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:13,870 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:13,870 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:13,870 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:13,872 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:13,872 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:13,872 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:13,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:13,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33370 deadline: 1730983333962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:13,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:13,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:13,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:13,974 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:13,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33346 deadline: 1730983333971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:13,974 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:13,975 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:13,975 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:13,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33368 deadline: 1730983333972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:13,975 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:13,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:13,975 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:13,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33340 deadline: 1730983333972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:13,975 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:13,975 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:13,976 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:13,976 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:13,976 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:13,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:13,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33334 deadline: 1730983333982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:13,991 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:13,992 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:13,992 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:14,004 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/3e17a8db69e04f15bf88f3720ea288e9 2024-11-07T12:41:14,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/bd4a3f5c2d754d499398190dd77aef9f is 50, key is test_row_0/C:col10/1730983272487/Put/seqid=0 2024-11-07T12:41:14,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741866_1042 (size=12151) 2024-11-07T12:41:14,087 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/bd4a3f5c2d754d499398190dd77aef9f 2024-11-07T12:41:14,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/4d7e0508c5014a4a8c167717b99a6336 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/4d7e0508c5014a4a8c167717b99a6336 2024-11-07T12:41:14,135 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/4d7e0508c5014a4a8c167717b99a6336, entries=150, sequenceid=175, filesize=11.9 K 2024-11-07T12:41:14,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/3e17a8db69e04f15bf88f3720ea288e9 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/3e17a8db69e04f15bf88f3720ea288e9 2024-11-07T12:41:14,155 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/3e17a8db69e04f15bf88f3720ea288e9, entries=150, sequenceid=175, filesize=11.9 K 2024-11-07T12:41:14,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/bd4a3f5c2d754d499398190dd77aef9f as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/bd4a3f5c2d754d499398190dd77aef9f 2024-11-07T12:41:14,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:14,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33370 deadline: 1730983334172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:14,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:14,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:14,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:14,183 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/bd4a3f5c2d754d499398190dd77aef9f, entries=150, sequenceid=175, filesize=11.9 K 2024-11-07T12:41:14,190 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=147.60 KB/151140 for 13f50260f7a437ef753d61fda36692fe in 701ms, sequenceid=175, compaction requested=true 2024-11-07T12:41:14,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 13f50260f7a437ef753d61fda36692fe: 2024-11-07T12:41:14,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:14,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-07T12:41:14,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-07T12:41:14,200 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 13f50260f7a437ef753d61fda36692fe 3/3 column families, dataSize=161.02 KB heapSize=441.23 KB 2024-11-07T12:41:14,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 13f50260f7a437ef753d61fda36692fe 2024-11-07T12:41:14,202 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-07T12:41:14,202 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5000 sec 2024-11-07T12:41:14,207 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 2.5210 sec 2024-11-07T12:41:14,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13f50260f7a437ef753d61fda36692fe, store=A 2024-11-07T12:41:14,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:41:14,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13f50260f7a437ef753d61fda36692fe, store=B 2024-11-07T12:41:14,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:41:14,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13f50260f7a437ef753d61fda36692fe, store=C 2024-11-07T12:41:14,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:41:14,221 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/de614f860f6243329267ec3c07070732 is 50, key is test_row_0/A:col10/1730983273809/Put/seqid=0 2024-11-07T12:41:14,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:14,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33340 deadline: 1730983334214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:14,224 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:14,224 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:14,224 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:14,224 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:14,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33368 deadline: 1730983334214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:14,225 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:14,225 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:14,225 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:14,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:14,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33346 deadline: 1730983334217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:14,226 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:14,226 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:14,227 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:14,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:14,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33334 deadline: 1730983334225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:14,230 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:14,230 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:14,230 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:14,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741867_1043 (size=14541) 2024-11-07T12:41:14,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:14,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33346 deadline: 1730983334334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:14,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:14,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33340 deadline: 1730983334334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:14,353 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:14,353 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:14,353 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:14,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:14,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33368 deadline: 1730983334333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:14,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:14,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33334 deadline: 1730983334335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:14,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:14,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:14,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:14,354 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:14,354 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:14,355 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:14,355 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:14,355 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:14,355 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:14,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:14,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33370 deadline: 1730983334485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:14,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:14,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:14,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:14,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:14,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33340 deadline: 1730983334563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:14,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:14,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33346 deadline: 1730983334563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:14,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:14,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:14,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33368 deadline: 1730983334563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:14,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:14,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:14,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:14,571 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:14,571 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:14,572 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:14,572 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:14,572 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:14,572 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:14,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33334 deadline: 1730983334563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:14,576 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:14,576 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:14,576 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:14,694 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/de614f860f6243329267ec3c07070732 2024-11-07T12:41:14,727 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/bc1a7d43d2784fefbdb460727bdffc41 is 50, key is test_row_0/B:col10/1730983273809/Put/seqid=0 2024-11-07T12:41:14,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741868_1044 (size=12151) 2024-11-07T12:41:14,767 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/bc1a7d43d2784fefbdb460727bdffc41 2024-11-07T12:41:14,803 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/9af50e3fcf4d4761bb6f701b1354e21a is 50, key is test_row_0/C:col10/1730983273809/Put/seqid=0 2024-11-07T12:41:14,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741869_1045 (size=12151) 2024-11-07T12:41:14,824 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/9af50e3fcf4d4761bb6f701b1354e21a 2024-11-07T12:41:14,846 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/de614f860f6243329267ec3c07070732 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/de614f860f6243329267ec3c07070732 2024-11-07T12:41:14,870 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/de614f860f6243329267ec3c07070732, entries=200, sequenceid=212, filesize=14.2 K 2024-11-07T12:41:14,872 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/bc1a7d43d2784fefbdb460727bdffc41 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/bc1a7d43d2784fefbdb460727bdffc41 2024-11-07T12:41:14,881 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/bc1a7d43d2784fefbdb460727bdffc41, entries=150, sequenceid=212, filesize=11.9 K 2024-11-07T12:41:14,886 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/9af50e3fcf4d4761bb6f701b1354e21a as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/9af50e3fcf4d4761bb6f701b1354e21a 2024-11-07T12:41:14,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:14,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33340 deadline: 1730983334882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:14,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:14,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33346 deadline: 1730983334884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:14,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:14,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33368 deadline: 1730983334884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:14,897 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:14,897 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:14,897 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:14,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:14,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:14,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:14,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:14,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:14,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:14,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:14,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33334 deadline: 1730983334882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:14,902 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:14,902 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:14,902 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:14,902 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/9af50e3fcf4d4761bb6f701b1354e21a, entries=150, sequenceid=212, filesize=11.9 K 2024-11-07T12:41:14,904 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~461.11 KB/472176, currentSize=20.13 KB/20610 for 13f50260f7a437ef753d61fda36692fe in 704ms, sequenceid=212, compaction requested=true 2024-11-07T12:41:14,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 13f50260f7a437ef753d61fda36692fe: 2024-11-07T12:41:14,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13f50260f7a437ef753d61fda36692fe:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:41:14,904 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:41:14,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:41:14,904 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:41:14,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13f50260f7a437ef753d61fda36692fe:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:41:14,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:41:14,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13f50260f7a437ef753d61fda36692fe:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:41:14,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:41:14,908 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48744 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:41:14,908 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 13f50260f7a437ef753d61fda36692fe/B is initiating minor compaction (all files) 2024-11-07T12:41:14,908 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13f50260f7a437ef753d61fda36692fe/B in TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:14,908 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55914 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:41:14,909 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/2ab0eb7528cd40a0b1ad4741e4353362, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/e78b648d4c224c56b73e4ee612eb63af, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/3e17a8db69e04f15bf88f3720ea288e9, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/bc1a7d43d2784fefbdb460727bdffc41] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp, totalSize=47.6 K 2024-11-07T12:41:14,909 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): 13f50260f7a437ef753d61fda36692fe/A is initiating minor compaction (all files) 2024-11-07T12:41:14,909 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13f50260f7a437ef753d61fda36692fe/A in TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:14,909 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/68784e5391194706a714e051045ee786, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/3ffc455d920d4467a2c74ed7a9e9da38, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/4d7e0508c5014a4a8c167717b99a6336, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/de614f860f6243329267ec3c07070732] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp, totalSize=54.6 K 2024-11-07T12:41:14,910 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ab0eb7528cd40a0b1ad4741e4353362, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1730983270777 2024-11-07T12:41:14,910 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68784e5391194706a714e051045ee786, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1730983270777 2024-11-07T12:41:14,910 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting e78b648d4c224c56b73e4ee612eb63af, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1730983271964 2024-11-07T12:41:14,911 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ffc455d920d4467a2c74ed7a9e9da38, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1730983271964 2024-11-07T12:41:14,911 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e17a8db69e04f15bf88f3720ea288e9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1730983272434 2024-11-07T12:41:14,912 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d7e0508c5014a4a8c167717b99a6336, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1730983272434 2024-11-07T12:41:14,912 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting de614f860f6243329267ec3c07070732, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1730983273809 2024-11-07T12:41:14,913 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting bc1a7d43d2784fefbdb460727bdffc41, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1730983273809 2024-11-07T12:41:14,959 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13f50260f7a437ef753d61fda36692fe#B#compaction#34 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:41:14,960 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/b595992aea684f65ba822ea3c929a2a5 is 50, key is test_row_0/B:col10/1730983273809/Put/seqid=0 2024-11-07T12:41:14,961 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13f50260f7a437ef753d61fda36692fe#A#compaction#33 average throughput is 0.39 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:41:14,962 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/e13948cacc374efc966cefd5e6dd7207 is 50, key is test_row_0/A:col10/1730983273809/Put/seqid=0 2024-11-07T12:41:15,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741870_1046 (size=12527) 2024-11-07T12:41:15,028 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/e13948cacc374efc966cefd5e6dd7207 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/e13948cacc374efc966cefd5e6dd7207 2024-11-07T12:41:15,038 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 13f50260f7a437ef753d61fda36692fe/A of 13f50260f7a437ef753d61fda36692fe into e13948cacc374efc966cefd5e6dd7207(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:41:15,038 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13f50260f7a437ef753d61fda36692fe: 2024-11-07T12:41:15,038 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., storeName=13f50260f7a437ef753d61fda36692fe/A, priority=12, startTime=1730983274904; duration=0sec 2024-11-07T12:41:15,038 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:41:15,038 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13f50260f7a437ef753d61fda36692fe:A 2024-11-07T12:41:15,038 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:41:15,041 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48744 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:41:15,041 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): 13f50260f7a437ef753d61fda36692fe/C is initiating minor compaction (all files) 2024-11-07T12:41:15,041 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13f50260f7a437ef753d61fda36692fe/C in TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:15,041 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/0f457255476e43daa9c9065a2bef4730, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/310e2a7ec25c41b99df1f6e06f8dff7c, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/bd4a3f5c2d754d499398190dd77aef9f, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/9af50e3fcf4d4761bb6f701b1354e21a] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp, totalSize=47.6 K 2024-11-07T12:41:15,042 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f457255476e43daa9c9065a2bef4730, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1730983270777 2024-11-07T12:41:15,043 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 310e2a7ec25c41b99df1f6e06f8dff7c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1730983271964 2024-11-07T12:41:15,043 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd4a3f5c2d754d499398190dd77aef9f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1730983272434 2024-11-07T12:41:15,044 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9af50e3fcf4d4761bb6f701b1354e21a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1730983273809 2024-11-07T12:41:15,045 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 13f50260f7a437ef753d61fda36692fe 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-07T12:41:15,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13f50260f7a437ef753d61fda36692fe, store=A 2024-11-07T12:41:15,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:15,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13f50260f7a437ef753d61fda36692fe, store=B 2024-11-07T12:41:15,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:15,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13f50260f7a437ef753d61fda36692fe, store=C 2024-11-07T12:41:15,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:15,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 13f50260f7a437ef753d61fda36692fe 2024-11-07T12:41:15,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741871_1047 (size=12527) 2024-11-07T12:41:15,063 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/9f3166eb5b794decbdd50fdb9e6c0230 is 50, key is test_row_0/A:col10/1730983275021/Put/seqid=0 2024-11-07T12:41:15,068 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13f50260f7a437ef753d61fda36692fe#C#compaction#36 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:41:15,068 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/b3722eb306b7474f94a13cb5c7475358 is 50, key is test_row_0/C:col10/1730983273809/Put/seqid=0 2024-11-07T12:41:15,073 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/b595992aea684f65ba822ea3c929a2a5 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/b595992aea684f65ba822ea3c929a2a5 2024-11-07T12:41:15,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741872_1048 (size=9757) 2024-11-07T12:41:15,085 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 13f50260f7a437ef753d61fda36692fe/B of 13f50260f7a437ef753d61fda36692fe into b595992aea684f65ba822ea3c929a2a5(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:41:15,085 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13f50260f7a437ef753d61fda36692fe: 2024-11-07T12:41:15,085 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., storeName=13f50260f7a437ef753d61fda36692fe/B, priority=12, startTime=1730983274904; duration=0sec 2024-11-07T12:41:15,085 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:41:15,085 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13f50260f7a437ef753d61fda36692fe:B 2024-11-07T12:41:15,087 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/9f3166eb5b794decbdd50fdb9e6c0230 2024-11-07T12:41:15,131 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/c539739013124ba689aece438ba3006b is 50, key is test_row_0/B:col10/1730983275021/Put/seqid=0 2024-11-07T12:41:15,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741873_1049 (size=12527) 2024-11-07T12:41:15,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741874_1050 (size=9757) 2024-11-07T12:41:15,175 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/c539739013124ba689aece438ba3006b 2024-11-07T12:41:15,210 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/684e250660ed4cb08608dc0b81c057b3 is 50, key is test_row_0/C:col10/1730983275021/Put/seqid=0 2024-11-07T12:41:15,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741875_1051 (size=9757) 2024-11-07T12:41:15,315 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1730983275157, max timestamp=1730983275281Num uniques 859; 2024-11-07T12:41:15,315 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-07T12:41:15,317 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1730983275157, max timestamp=1730983275281Num uniques 859; 2024-11-07T12:41:15,318 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-07T12:41:15,320 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1730983275157, max timestamp=1730983275281Num uniques 859; 2024-11-07T12:41:15,320 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-07T12:41:15,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:15,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33334 deadline: 1730983335436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:15,457 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:15,457 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:15,457 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:15,463 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1730983275383, max timestamp=1730983275450Num uniques 1018; 2024-11-07T12:41:15,464 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=14558, suffix heap size=133840,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=318 2024-11-07T12:41:15,464 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1730983275383, max timestamp=1730983275450Num uniques 1018; 2024-11-07T12:41:15,465 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=14558, suffix heap size=133840,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=318 2024-11-07T12:41:15,467 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1730983275383, max timestamp=1730983275450Num uniques 1018; 2024-11-07T12:41:15,467 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=14558, suffix heap size=133840,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=318 2024-11-07T12:41:15,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:15,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33346 deadline: 1730983335521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:15,539 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:15,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:15,539 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:15,539 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:15,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33368 deadline: 1730983335521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:15,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:15,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:15,540 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:15,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:15,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33370 deadline: 1730983335531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:15,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:15,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:15,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:15,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:15,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33340 deadline: 1730983335538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:15,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:15,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:15,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:15,570 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/b3722eb306b7474f94a13cb5c7475358 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/b3722eb306b7474f94a13cb5c7475358 2024-11-07T12:41:15,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:15,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33334 deadline: 1730983335565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:15,578 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:15,578 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:15,579 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:15,582 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 13f50260f7a437ef753d61fda36692fe/C of 13f50260f7a437ef753d61fda36692fe into b3722eb306b7474f94a13cb5c7475358(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:41:15,582 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13f50260f7a437ef753d61fda36692fe: 2024-11-07T12:41:15,582 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., storeName=13f50260f7a437ef753d61fda36692fe/C, priority=12, startTime=1730983274905; duration=0sec 2024-11-07T12:41:15,582 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:41:15,582 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13f50260f7a437ef753d61fda36692fe:C 2024-11-07T12:41:15,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:15,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33368 deadline: 1730983335643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:15,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:15,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:15,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:15,648 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/684e250660ed4cb08608dc0b81c057b3 2024-11-07T12:41:15,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:15,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33346 deadline: 1730983335643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:15,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:15,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:15,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:15,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:15,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33370 deadline: 1730983335652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:15,676 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/9f3166eb5b794decbdd50fdb9e6c0230 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/9f3166eb5b794decbdd50fdb9e6c0230 2024-11-07T12:41:15,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:15,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:15,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:15,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:15,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33340 deadline: 1730983335662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:15,685 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:15,685 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:15,685 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:15,706 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/9f3166eb5b794decbdd50fdb9e6c0230, entries=100, sequenceid=223, filesize=9.5 K 2024-11-07T12:41:15,725 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/c539739013124ba689aece438ba3006b as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/c539739013124ba689aece438ba3006b 2024-11-07T12:41:15,784 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/c539739013124ba689aece438ba3006b, entries=100, sequenceid=223, filesize=9.5 K 2024-11-07T12:41:15,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/684e250660ed4cb08608dc0b81c057b3 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/684e250660ed4cb08608dc0b81c057b3 2024-11-07T12:41:15,795 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/684e250660ed4cb08608dc0b81c057b3, entries=100, sequenceid=223, filesize=9.5 K 2024-11-07T12:41:15,796 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 13f50260f7a437ef753d61fda36692fe in 751ms, sequenceid=223, compaction requested=false 2024-11-07T12:41:15,796 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 13f50260f7a437ef753d61fda36692fe: 2024-11-07T12:41:15,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 13f50260f7a437ef753d61fda36692fe 2024-11-07T12:41:15,820 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 13f50260f7a437ef753d61fda36692fe 3/3 column families, dataSize=147.60 KB heapSize=409.92 KB 2024-11-07T12:41:15,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13f50260f7a437ef753d61fda36692fe, store=A 2024-11-07T12:41:15,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:41:15,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13f50260f7a437ef753d61fda36692fe, store=B 2024-11-07T12:41:15,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:41:15,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13f50260f7a437ef753d61fda36692fe, store=C 2024-11-07T12:41:15,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:41:15,850 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/5c8d99db3f8149b88db0493a97149117 is 50, key is test_row_0/A:col10/1730983275535/Put/seqid=0 2024-11-07T12:41:15,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-07T12:41:15,862 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:41:15,864 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:41:15,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-07T12:41:15,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-07T12:41:15,876 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:41:15,878 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:41:15,878 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:41:15,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:15,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33370 deadline: 1730983335886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:15,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:15,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:15,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:15,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:15,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33346 deadline: 1730983335889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:15,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:15,911 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:15,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33340 deadline: 1730983335892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:15,911 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:15,911 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:15,911 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:15,911 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:15,911 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:15,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:15,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33368 deadline: 1730983335899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:15,921 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:15,921 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:15,921 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:15,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:15,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33334 deadline: 1730983335899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:15,922 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:15,922 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:15,922 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:15,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741876_1052 (size=14741) 2024-11-07T12:41:15,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-07T12:41:16,042 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-07T12:41:16,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:16,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. as already flushing 2024-11-07T12:41:16,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:16,042 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:16,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:16,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:16,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:16,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33368 deadline: 1730983336033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:16,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:16,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33334 deadline: 1730983336032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:16,055 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:16,055 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:16,055 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:16,055 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:16,055 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:16,055 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:16,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-07T12:41:16,198 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-07T12:41:16,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:16,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. as already flushing 2024-11-07T12:41:16,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:16,198 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:16,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:16,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:16,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:16,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:16,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33346 deadline: 1730983336222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:16,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33370 deadline: 1730983336222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:16,244 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:16,245 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:16,245 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:16,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:16,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:16,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:16,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:16,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33340 deadline: 1730983336222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:16,246 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:16,246 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:16,246 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:16,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:16,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33368 deadline: 1730983336262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:16,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:16,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:16,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:16,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:16,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33334 deadline: 1730983336262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:16,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:16,283 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:16,283 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:16,324 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/5c8d99db3f8149b88db0493a97149117 2024-11-07T12:41:16,355 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-07T12:41:16,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:16,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. as already flushing 2024-11-07T12:41:16,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:16,361 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:16,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:16,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:16,366 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/cb8ade718bc446c388991c12fa5d4dc1 is 50, key is test_row_0/B:col10/1730983275535/Put/seqid=0 2024-11-07T12:41:16,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741877_1053 (size=12301) 2024-11-07T12:41:16,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-07T12:41:16,518 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-07T12:41:16,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:16,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. as already flushing 2024-11-07T12:41:16,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:16,519 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:16,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:16,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:16,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:16,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33368 deadline: 1730983336583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:16,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:16,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:16,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:16,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:16,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33334 deadline: 1730983336591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:16,600 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:16,600 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:16,600 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:16,678 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-07T12:41:16,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:16,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. as already flushing 2024-11-07T12:41:16,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:16,679 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:16,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:16,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:16,753 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:16,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33346 deadline: 1730983336751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:16,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:16,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33340 deadline: 1730983336753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:16,756 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:16,756 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:16,756 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:16,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:16,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33370 deadline: 1730983336755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:16,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:16,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:16,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:16,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:16,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:16,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:16,816 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/cb8ade718bc446c388991c12fa5d4dc1 2024-11-07T12:41:16,833 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-07T12:41:16,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:16,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. as already flushing 2024-11-07T12:41:16,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:16,834 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:16,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:16,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:16,852 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/1b510c1251844884b3e845483d058bba is 50, key is test_row_0/C:col10/1730983275535/Put/seqid=0 2024-11-07T12:41:16,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741878_1054 (size=12301) 2024-11-07T12:41:16,991 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-07T12:41:16,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:16,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. as already flushing 2024-11-07T12:41:16,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:16,992 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:16,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:16,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:17,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-07T12:41:17,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:17,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33368 deadline: 1730983337105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:17,111 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:17,111 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:17,111 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:17,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:17,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33334 deadline: 1730983337105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:17,113 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:17,113 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:17,113 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:17,151 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-07T12:41:17,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:17,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. as already flushing 2024-11-07T12:41:17,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:17,152 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:17,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:17,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:17,313 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-07T12:41:17,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:17,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. as already flushing 2024-11-07T12:41:17,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:17,317 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:17,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:17,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:17,322 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/1b510c1251844884b3e845483d058bba 2024-11-07T12:41:17,352 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/5c8d99db3f8149b88db0493a97149117 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/5c8d99db3f8149b88db0493a97149117 2024-11-07T12:41:17,361 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/5c8d99db3f8149b88db0493a97149117, entries=200, sequenceid=278, filesize=14.4 K 2024-11-07T12:41:17,363 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/cb8ade718bc446c388991c12fa5d4dc1 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/cb8ade718bc446c388991c12fa5d4dc1 2024-11-07T12:41:17,383 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/cb8ade718bc446c388991c12fa5d4dc1, entries=150, sequenceid=278, filesize=12.0 K 2024-11-07T12:41:17,386 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/1b510c1251844884b3e845483d058bba as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/1b510c1251844884b3e845483d058bba 2024-11-07T12:41:17,409 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/1b510c1251844884b3e845483d058bba, entries=150, sequenceid=278, filesize=12.0 K 2024-11-07T12:41:17,417 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~409.88 KB/419712, currentSize=40.25 KB/41220 for 13f50260f7a437ef753d61fda36692fe in 1597ms, sequenceid=278, compaction requested=true 2024-11-07T12:41:17,417 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 13f50260f7a437ef753d61fda36692fe: 2024-11-07T12:41:17,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13f50260f7a437ef753d61fda36692fe:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:41:17,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:41:17,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13f50260f7a437ef753d61fda36692fe:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:41:17,418 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:41:17,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:41:17,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13f50260f7a437ef753d61fda36692fe:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:41:17,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-07T12:41:17,418 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:41:17,420 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37025 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:41:17,420 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): 13f50260f7a437ef753d61fda36692fe/A is initiating minor compaction (all files) 2024-11-07T12:41:17,420 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13f50260f7a437ef753d61fda36692fe/A in TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:17,420 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/e13948cacc374efc966cefd5e6dd7207, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/9f3166eb5b794decbdd50fdb9e6c0230, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/5c8d99db3f8149b88db0493a97149117] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp, totalSize=36.2 K 2024-11-07T12:41:17,420 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:41:17,421 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 13f50260f7a437ef753d61fda36692fe/B is initiating minor compaction (all files) 2024-11-07T12:41:17,421 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13f50260f7a437ef753d61fda36692fe/B in TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:17,421 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/b595992aea684f65ba822ea3c929a2a5, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/c539739013124ba689aece438ba3006b, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/cb8ade718bc446c388991c12fa5d4dc1] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp, totalSize=33.8 K 2024-11-07T12:41:17,421 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting e13948cacc374efc966cefd5e6dd7207, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1730983273809 2024-11-07T12:41:17,421 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting b595992aea684f65ba822ea3c929a2a5, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1730983273809 2024-11-07T12:41:17,422 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f3166eb5b794decbdd50fdb9e6c0230, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1730983275021 2024-11-07T12:41:17,423 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting c539739013124ba689aece438ba3006b, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1730983275021 2024-11-07T12:41:17,423 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting cb8ade718bc446c388991c12fa5d4dc1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1730983275527 2024-11-07T12:41:17,424 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c8d99db3f8149b88db0493a97149117, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1730983275513 2024-11-07T12:41:17,442 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13f50260f7a437ef753d61fda36692fe#B#compaction#42 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:41:17,443 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/b9479869f2bc455fa3833a74b205f591 is 50, key is test_row_0/B:col10/1730983275535/Put/seqid=0 2024-11-07T12:41:17,451 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13f50260f7a437ef753d61fda36692fe#A#compaction#43 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:41:17,452 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/ba5ac0ad4d8a4542934813bc04ca6b5a is 50, key is test_row_0/A:col10/1730983275535/Put/seqid=0 2024-11-07T12:41:17,477 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-07T12:41:17,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:17,478 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 13f50260f7a437ef753d61fda36692fe 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-07T12:41:17,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13f50260f7a437ef753d61fda36692fe, store=A 2024-11-07T12:41:17,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:17,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13f50260f7a437ef753d61fda36692fe, store=B 2024-11-07T12:41:17,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:17,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13f50260f7a437ef753d61fda36692fe, store=C 2024-11-07T12:41:17,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:17,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/f910199589ed409794c0442f2583e62f is 50, key is test_row_0/A:col10/1730983275865/Put/seqid=0 2024-11-07T12:41:17,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741880_1056 (size=12779) 2024-11-07T12:41:17,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741879_1055 (size=12779) 2024-11-07T12:41:17,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741881_1057 (size=12301) 2024-11-07T12:41:17,544 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/f910199589ed409794c0442f2583e62f 2024-11-07T12:41:17,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/a3674a1e469d466f9c3947511e7cf8d3 is 50, key is test_row_0/B:col10/1730983275865/Put/seqid=0 2024-11-07T12:41:17,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741882_1058 (size=12301) 2024-11-07T12:41:17,647 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/a3674a1e469d466f9c3947511e7cf8d3 2024-11-07T12:41:17,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/c8f233088adb46d99f0c78e90497b035 is 50, key is test_row_0/C:col10/1730983275865/Put/seqid=0 2024-11-07T12:41:17,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741883_1059 (size=12301) 2024-11-07T12:41:17,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 13f50260f7a437ef753d61fda36692fe 2024-11-07T12:41:17,763 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. as already flushing 2024-11-07T12:41:17,813 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1730983277778, max timestamp=1730983277809Num uniques 859; 2024-11-07T12:41:17,813 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-07T12:41:17,813 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1730983277778, max timestamp=1730983277809Num uniques 859; 2024-11-07T12:41:17,813 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1730983277778, max timestamp=1730983277809Num uniques 859; 2024-11-07T12:41:17,813 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-07T12:41:17,813 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-07T12:41:17,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:17,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33346 deadline: 1730983337855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:17,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:17,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33370 deadline: 1730983337856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:17,862 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:17,862 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:17,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:17,862 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:17,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:17,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:17,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:17,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33340 deadline: 1730983337858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:17,863 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:17,863 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:17,863 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:17,929 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/b9479869f2bc455fa3833a74b205f591 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/b9479869f2bc455fa3833a74b205f591 2024-11-07T12:41:17,930 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/ba5ac0ad4d8a4542934813bc04ca6b5a as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/ba5ac0ad4d8a4542934813bc04ca6b5a 2024-11-07T12:41:17,939 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13f50260f7a437ef753d61fda36692fe/B of 13f50260f7a437ef753d61fda36692fe into b9479869f2bc455fa3833a74b205f591(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:41:17,939 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13f50260f7a437ef753d61fda36692fe: 2024-11-07T12:41:17,939 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., storeName=13f50260f7a437ef753d61fda36692fe/B, priority=13, startTime=1730983277418; duration=0sec 2024-11-07T12:41:17,939 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13f50260f7a437ef753d61fda36692fe/A of 13f50260f7a437ef753d61fda36692fe into ba5ac0ad4d8a4542934813bc04ca6b5a(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:41:17,939 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13f50260f7a437ef753d61fda36692fe: 2024-11-07T12:41:17,939 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., storeName=13f50260f7a437ef753d61fda36692fe/A, priority=13, startTime=1730983277417; duration=0sec 2024-11-07T12:41:17,939 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:41:17,939 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13f50260f7a437ef753d61fda36692fe:B 2024-11-07T12:41:17,939 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:41:17,939 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:41:17,939 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13f50260f7a437ef753d61fda36692fe:A 2024-11-07T12:41:17,944 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:41:17,944 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 13f50260f7a437ef753d61fda36692fe/C is initiating minor compaction (all files) 2024-11-07T12:41:17,944 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13f50260f7a437ef753d61fda36692fe/C in TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:17,944 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/b3722eb306b7474f94a13cb5c7475358, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/684e250660ed4cb08608dc0b81c057b3, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/1b510c1251844884b3e845483d058bba] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp, totalSize=33.8 K 2024-11-07T12:41:17,945 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting b3722eb306b7474f94a13cb5c7475358, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1730983273809 2024-11-07T12:41:17,945 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 684e250660ed4cb08608dc0b81c057b3, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1730983275021 2024-11-07T12:41:17,946 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b510c1251844884b3e845483d058bba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1730983275527 2024-11-07T12:41:17,968 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13f50260f7a437ef753d61fda36692fe#C#compaction#47 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:41:17,969 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/7501b5b83cbe4d4b9ed84d9c42d3b1ec is 50, key is test_row_0/C:col10/1730983275535/Put/seqid=0 2024-11-07T12:41:17,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:17,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33346 deadline: 1730983337971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:17,977 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:17,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:17,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:17,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:17,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33340 deadline: 1730983337972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:17,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:17,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33370 deadline: 1730983337972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:17,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:17,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:17,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:17,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741884_1060 (size=12779) 2024-11-07T12:41:17,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:17,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:17,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:18,003 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/7501b5b83cbe4d4b9ed84d9c42d3b1ec as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/7501b5b83cbe4d4b9ed84d9c42d3b1ec 2024-11-07T12:41:18,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-07T12:41:18,045 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13f50260f7a437ef753d61fda36692fe/C of 13f50260f7a437ef753d61fda36692fe into 7501b5b83cbe4d4b9ed84d9c42d3b1ec(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:41:18,045 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13f50260f7a437ef753d61fda36692fe: 2024-11-07T12:41:18,045 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., storeName=13f50260f7a437ef753d61fda36692fe/C, priority=13, startTime=1730983277418; duration=0sec 2024-11-07T12:41:18,045 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:41:18,045 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13f50260f7a437ef753d61fda36692fe:C 2024-11-07T12:41:18,115 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/c8f233088adb46d99f0c78e90497b035 2024-11-07T12:41:18,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/f910199589ed409794c0442f2583e62f as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/f910199589ed409794c0442f2583e62f 2024-11-07T12:41:18,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:18,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33368 deadline: 1730983338122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:18,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:18,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33334 deadline: 1730983338122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:18,140 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to db9ad1cb6cf9,34293,1730983260217 for 'test_row_0' in 13f50260f7a437ef753d61fda36692fe of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2239 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:41:18,143 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to db9ad1cb6cf9,34293,1730983260217 for 'test_row_0' in 13f50260f7a437ef753d61fda36692fe of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2238 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:41:18,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:18,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:18,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:18,144 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:18,144 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:18,144 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:18,155 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/f910199589ed409794c0442f2583e62f, entries=150, sequenceid=287, filesize=12.0 K 2024-11-07T12:41:18,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/a3674a1e469d466f9c3947511e7cf8d3 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/a3674a1e469d466f9c3947511e7cf8d3 2024-11-07T12:41:18,170 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/a3674a1e469d466f9c3947511e7cf8d3, entries=150, sequenceid=287, filesize=12.0 K 2024-11-07T12:41:18,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/c8f233088adb46d99f0c78e90497b035 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/c8f233088adb46d99f0c78e90497b035 2024-11-07T12:41:18,180 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/c8f233088adb46d99f0c78e90497b035, entries=150, sequenceid=287, filesize=12.0 K 2024-11-07T12:41:18,182 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=147.60 KB/151140 for 13f50260f7a437ef753d61fda36692fe in 704ms, sequenceid=287, compaction requested=false 2024-11-07T12:41:18,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 13f50260f7a437ef753d61fda36692fe: 2024-11-07T12:41:18,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:18,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-07T12:41:18,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-07T12:41:18,189 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-07T12:41:18,190 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3080 sec 2024-11-07T12:41:18,192 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 13f50260f7a437ef753d61fda36692fe 3/3 column families, dataSize=154.31 KB heapSize=430.69 KB 2024-11-07T12:41:18,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13f50260f7a437ef753d61fda36692fe, store=A 2024-11-07T12:41:18,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:41:18,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13f50260f7a437ef753d61fda36692fe, store=B 2024-11-07T12:41:18,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:41:18,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13f50260f7a437ef753d61fda36692fe, store=C 2024-11-07T12:41:18,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:41:18,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 13f50260f7a437ef753d61fda36692fe 2024-11-07T12:41:18,196 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 2.3270 sec 2024-11-07T12:41:18,206 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/b0ededf8e5e24e5b86963104dfa8d905 is 50, key is test_row_0/A:col10/1730983277838/Put/seqid=0 2024-11-07T12:41:18,214 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:18,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33340 deadline: 1730983338205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:18,215 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:18,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33370 deadline: 1730983338207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:18,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:18,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:18,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:18,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:18,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:18,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:18,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:18,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33346 deadline: 1730983338215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:18,218 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:18,218 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:18,218 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:18,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741885_1061 (size=14741) 2024-11-07T12:41:18,324 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:18,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33340 deadline: 1730983338322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:18,326 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:18,326 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:18,326 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:18,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:18,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33370 deadline: 1730983338323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:18,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:18,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:18,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:18,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:18,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33346 deadline: 1730983338323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:18,332 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:18,333 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:18,333 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:18,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:18,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33340 deadline: 1730983338532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:18,540 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:18,540 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:18,541 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:18,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:18,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33370 deadline: 1730983338542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:18,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:18,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:18,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:18,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:18,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33346 deadline: 1730983338542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:18,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:18,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:18,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:18,650 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/b0ededf8e5e24e5b86963104dfa8d905 2024-11-07T12:41:18,671 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/b1d69d26b95d419c823ac2eb5fa2420c is 50, key is test_row_0/B:col10/1730983277838/Put/seqid=0 2024-11-07T12:41:18,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741886_1062 (size=12301) 2024-11-07T12:41:18,729 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/b1d69d26b95d419c823ac2eb5fa2420c 2024-11-07T12:41:18,759 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/95102599eaa74ca1b1a3d88f5b6ffc9e is 50, key is test_row_0/C:col10/1730983277838/Put/seqid=0 2024-11-07T12:41:18,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741887_1063 (size=12301) 2024-11-07T12:41:18,787 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/95102599eaa74ca1b1a3d88f5b6ffc9e 2024-11-07T12:41:18,805 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/b0ededf8e5e24e5b86963104dfa8d905 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/b0ededf8e5e24e5b86963104dfa8d905 2024-11-07T12:41:18,811 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/b0ededf8e5e24e5b86963104dfa8d905, entries=200, sequenceid=327, filesize=14.4 K 2024-11-07T12:41:18,812 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/b1d69d26b95d419c823ac2eb5fa2420c as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/b1d69d26b95d419c823ac2eb5fa2420c 2024-11-07T12:41:18,824 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/b1d69d26b95d419c823ac2eb5fa2420c, entries=150, sequenceid=327, filesize=12.0 K 2024-11-07T12:41:18,825 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/95102599eaa74ca1b1a3d88f5b6ffc9e as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/95102599eaa74ca1b1a3d88f5b6ffc9e 2024-11-07T12:41:18,834 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/95102599eaa74ca1b1a3d88f5b6ffc9e, entries=150, sequenceid=327, filesize=12.0 K 2024-11-07T12:41:18,836 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~430.64 KB/440976, currentSize=33.54 KB/34350 for 13f50260f7a437ef753d61fda36692fe in 643ms, sequenceid=327, compaction requested=true 2024-11-07T12:41:18,836 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 13f50260f7a437ef753d61fda36692fe: 2024-11-07T12:41:18,836 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:41:18,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13f50260f7a437ef753d61fda36692fe:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:41:18,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:41:18,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13f50260f7a437ef753d61fda36692fe:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:41:18,838 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:41:18,838 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39821 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:41:18,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:41:18,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13f50260f7a437ef753d61fda36692fe:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:41:18,840 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): 13f50260f7a437ef753d61fda36692fe/A is initiating minor compaction (all files) 2024-11-07T12:41:18,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:41:18,841 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13f50260f7a437ef753d61fda36692fe/A in TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:18,841 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/ba5ac0ad4d8a4542934813bc04ca6b5a, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/f910199589ed409794c0442f2583e62f, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/b0ededf8e5e24e5b86963104dfa8d905] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp, totalSize=38.9 K 2024-11-07T12:41:18,842 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba5ac0ad4d8a4542934813bc04ca6b5a, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1730983275527 2024-11-07T12:41:18,843 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting f910199589ed409794c0442f2583e62f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1730983275859 2024-11-07T12:41:18,844 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:41:18,844 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0ededf8e5e24e5b86963104dfa8d905, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1730983277838 2024-11-07T12:41:18,844 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 13f50260f7a437ef753d61fda36692fe/B is initiating minor compaction (all files) 2024-11-07T12:41:18,844 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13f50260f7a437ef753d61fda36692fe/B in TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:18,844 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/b9479869f2bc455fa3833a74b205f591, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/a3674a1e469d466f9c3947511e7cf8d3, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/b1d69d26b95d419c823ac2eb5fa2420c] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp, totalSize=36.5 K 2024-11-07T12:41:18,845 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting b9479869f2bc455fa3833a74b205f591, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1730983275527 2024-11-07T12:41:18,846 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting a3674a1e469d466f9c3947511e7cf8d3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1730983275859 2024-11-07T12:41:18,846 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting b1d69d26b95d419c823ac2eb5fa2420c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1730983277838 2024-11-07T12:41:18,862 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13f50260f7a437ef753d61fda36692fe#A#compaction#51 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:41:18,862 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13f50260f7a437ef753d61fda36692fe#B#compaction#52 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:41:18,863 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/e044e1b4dc6244d986037942bd6b684b is 50, key is test_row_0/B:col10/1730983277838/Put/seqid=0 2024-11-07T12:41:18,863 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/0b95ee000f9e4094bd0925b6de91945d is 50, key is test_row_0/A:col10/1730983277838/Put/seqid=0 2024-11-07T12:41:18,887 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 13f50260f7a437ef753d61fda36692fe 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-07T12:41:18,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 13f50260f7a437ef753d61fda36692fe 2024-11-07T12:41:18,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13f50260f7a437ef753d61fda36692fe, store=A 2024-11-07T12:41:18,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:18,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13f50260f7a437ef753d61fda36692fe, store=B 2024-11-07T12:41:18,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:18,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13f50260f7a437ef753d61fda36692fe, store=C 2024-11-07T12:41:18,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:18,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741888_1064 (size=12881) 2024-11-07T12:41:18,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741889_1065 (size=12881) 2024-11-07T12:41:18,934 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/12649c51fad447469a8f2912cf01e9b9 is 50, key is test_row_0/A:col10/1730983278862/Put/seqid=0 2024-11-07T12:41:18,942 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/e044e1b4dc6244d986037942bd6b684b as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/e044e1b4dc6244d986037942bd6b684b 2024-11-07T12:41:18,953 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13f50260f7a437ef753d61fda36692fe/B of 13f50260f7a437ef753d61fda36692fe into e044e1b4dc6244d986037942bd6b684b(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:41:18,953 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13f50260f7a437ef753d61fda36692fe: 2024-11-07T12:41:18,953 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., storeName=13f50260f7a437ef753d61fda36692fe/B, priority=13, startTime=1730983278838; duration=0sec 2024-11-07T12:41:18,954 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:41:18,954 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13f50260f7a437ef753d61fda36692fe:B 2024-11-07T12:41:18,954 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:41:18,958 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:41:18,958 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 13f50260f7a437ef753d61fda36692fe/C is initiating minor compaction (all files) 2024-11-07T12:41:18,958 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13f50260f7a437ef753d61fda36692fe/C in TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:18,959 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/7501b5b83cbe4d4b9ed84d9c42d3b1ec, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/c8f233088adb46d99f0c78e90497b035, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/95102599eaa74ca1b1a3d88f5b6ffc9e] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp, totalSize=36.5 K 2024-11-07T12:41:18,960 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 7501b5b83cbe4d4b9ed84d9c42d3b1ec, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1730983275527 2024-11-07T12:41:18,960 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting c8f233088adb46d99f0c78e90497b035, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1730983275859 2024-11-07T12:41:18,961 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 95102599eaa74ca1b1a3d88f5b6ffc9e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1730983277838 2024-11-07T12:41:18,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741890_1066 (size=17181) 2024-11-07T12:41:18,977 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/12649c51fad447469a8f2912cf01e9b9 2024-11-07T12:41:18,988 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13f50260f7a437ef753d61fda36692fe#C#compaction#54 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:41:18,989 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/5664d20731f7454fb1a564e7129d21a0 is 50, key is test_row_0/C:col10/1730983277838/Put/seqid=0 2024-11-07T12:41:18,999 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/2a4b1e6147f34934827d47d133a5ad5c is 50, key is test_row_0/B:col10/1730983278862/Put/seqid=0 2024-11-07T12:41:19,004 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1730983278967, max timestamp=1730983278996Num uniques 859; 2024-11-07T12:41:19,004 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-07T12:41:19,005 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1730983278967, max timestamp=1730983278997Num uniques 859; 2024-11-07T12:41:19,005 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-07T12:41:19,005 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1730983278967, max timestamp=1730983279001Num uniques 859; 2024-11-07T12:41:19,005 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-07T12:41:19,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741891_1067 (size=12881) 2024-11-07T12:41:19,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741892_1068 (size=12301) 2024-11-07T12:41:19,063 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/2a4b1e6147f34934827d47d133a5ad5c 2024-11-07T12:41:19,076 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/85083acf838440a1898c59406e57ee83 is 50, key is test_row_0/C:col10/1730983278862/Put/seqid=0 2024-11-07T12:41:19,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:19,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33340 deadline: 1730983339075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:19,089 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:19,089 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:19,089 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:19,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:19,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33370 deadline: 1730983339078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:19,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:19,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:19,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:19,094 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:19,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33346 deadline: 1730983339089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:19,095 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:19,095 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:19,095 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:19,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741893_1069 (size=12301) 2024-11-07T12:41:19,116 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/85083acf838440a1898c59406e57ee83 2024-11-07T12:41:19,124 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/12649c51fad447469a8f2912cf01e9b9 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/12649c51fad447469a8f2912cf01e9b9 2024-11-07T12:41:19,133 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/12649c51fad447469a8f2912cf01e9b9, entries=250, sequenceid=338, filesize=16.8 K 2024-11-07T12:41:19,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/2a4b1e6147f34934827d47d133a5ad5c as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/2a4b1e6147f34934827d47d133a5ad5c 2024-11-07T12:41:19,153 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/2a4b1e6147f34934827d47d133a5ad5c, entries=150, sequenceid=338, filesize=12.0 K 2024-11-07T12:41:19,155 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/85083acf838440a1898c59406e57ee83 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/85083acf838440a1898c59406e57ee83 2024-11-07T12:41:19,160 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/85083acf838440a1898c59406e57ee83, entries=150, sequenceid=338, filesize=12.0 K 2024-11-07T12:41:19,162 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 13f50260f7a437ef753d61fda36692fe in 274ms, sequenceid=338, compaction requested=false 2024-11-07T12:41:19,162 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 13f50260f7a437ef753d61fda36692fe: 2024-11-07T12:41:19,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 13f50260f7a437ef753d61fda36692fe 2024-11-07T12:41:19,199 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 13f50260f7a437ef753d61fda36692fe 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-07T12:41:19,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13f50260f7a437ef753d61fda36692fe, store=A 2024-11-07T12:41:19,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:41:19,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13f50260f7a437ef753d61fda36692fe, store=B 2024-11-07T12:41:19,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:41:19,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13f50260f7a437ef753d61fda36692fe, store=C 2024-11-07T12:41:19,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:41:19,223 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/a0c29527ec0140c1be1f528b2bd81cb9 is 50, key is test_row_0/A:col10/1730983279197/Put/seqid=0 2024-11-07T12:41:19,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:19,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33370 deadline: 1730983339239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:19,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:19,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:19,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33340 deadline: 1730983339240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:19,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:19,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:19,248 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:19,248 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:19,248 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:19,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:19,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33346 deadline: 1730983339247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:19,252 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:19,252 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:19,253 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:19,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741894_1070 (size=14741) 2024-11-07T12:41:19,278 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/a0c29527ec0140c1be1f528b2bd81cb9 2024-11-07T12:41:19,304 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/d06109e82b9f4f25badbfd68bd33ae0c is 50, key is test_row_0/B:col10/1730983279197/Put/seqid=0 2024-11-07T12:41:19,325 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/0b95ee000f9e4094bd0925b6de91945d as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/0b95ee000f9e4094bd0925b6de91945d 2024-11-07T12:41:19,337 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13f50260f7a437ef753d61fda36692fe/A of 13f50260f7a437ef753d61fda36692fe into 0b95ee000f9e4094bd0925b6de91945d(size=12.6 K), total size for store is 29.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:41:19,337 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13f50260f7a437ef753d61fda36692fe: 2024-11-07T12:41:19,337 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., storeName=13f50260f7a437ef753d61fda36692fe/A, priority=13, startTime=1730983278836; duration=0sec 2024-11-07T12:41:19,337 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:41:19,337 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13f50260f7a437ef753d61fda36692fe:A 2024-11-07T12:41:19,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:19,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33370 deadline: 1730983339351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:19,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:19,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33340 deadline: 1730983339352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:19,360 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:19,360 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:19,360 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:19,360 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:19,360 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:19,361 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:19,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:19,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33346 deadline: 1730983339362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:19,363 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:19,363 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:19,363 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:19,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741895_1071 (size=12301) 2024-11-07T12:41:19,369 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/d06109e82b9f4f25badbfd68bd33ae0c 2024-11-07T12:41:19,387 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/e206dd985be74ba3a7c8f4c49b0d7227 is 50, key is test_row_0/C:col10/1730983279197/Put/seqid=0 2024-11-07T12:41:19,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741896_1072 (size=12301) 2024-11-07T12:41:19,401 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/e206dd985be74ba3a7c8f4c49b0d7227 2024-11-07T12:41:19,417 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/a0c29527ec0140c1be1f528b2bd81cb9 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/a0c29527ec0140c1be1f528b2bd81cb9 2024-11-07T12:41:19,424 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/a0c29527ec0140c1be1f528b2bd81cb9, entries=200, sequenceid=376, filesize=14.4 K 2024-11-07T12:41:19,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/d06109e82b9f4f25badbfd68bd33ae0c as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/d06109e82b9f4f25badbfd68bd33ae0c 2024-11-07T12:41:19,443 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/d06109e82b9f4f25badbfd68bd33ae0c, entries=150, sequenceid=376, filesize=12.0 K 2024-11-07T12:41:19,445 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/e206dd985be74ba3a7c8f4c49b0d7227 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/e206dd985be74ba3a7c8f4c49b0d7227 2024-11-07T12:41:19,454 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/e206dd985be74ba3a7c8f4c49b0d7227, entries=150, sequenceid=376, filesize=12.0 K 2024-11-07T12:41:19,456 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 13f50260f7a437ef753d61fda36692fe in 256ms, sequenceid=376, compaction requested=true 2024-11-07T12:41:19,456 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 13f50260f7a437ef753d61fda36692fe: 2024-11-07T12:41:19,456 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:41:19,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13f50260f7a437ef753d61fda36692fe:A, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:41:19,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:41:19,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13f50260f7a437ef753d61fda36692fe:B, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:41:19,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:41:19,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13f50260f7a437ef753d61fda36692fe:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:41:19,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-07T12:41:19,460 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 44803 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:41:19,460 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): 13f50260f7a437ef753d61fda36692fe/A is initiating minor compaction (all files) 2024-11-07T12:41:19,460 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13f50260f7a437ef753d61fda36692fe/A in TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:19,460 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/0b95ee000f9e4094bd0925b6de91945d, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/12649c51fad447469a8f2912cf01e9b9, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/a0c29527ec0140c1be1f528b2bd81cb9] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp, totalSize=43.8 K 2024-11-07T12:41:19,460 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b95ee000f9e4094bd0925b6de91945d, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1730983277838 2024-11-07T12:41:19,461 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12649c51fad447469a8f2912cf01e9b9, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1730983278202 2024-11-07T12:41:19,462 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0c29527ec0140c1be1f528b2bd81cb9, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1730983279049 2024-11-07T12:41:19,463 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/5664d20731f7454fb1a564e7129d21a0 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/5664d20731f7454fb1a564e7129d21a0 2024-11-07T12:41:19,479 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13f50260f7a437ef753d61fda36692fe/C of 13f50260f7a437ef753d61fda36692fe into 5664d20731f7454fb1a564e7129d21a0(size=12.6 K), total size for store is 36.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:41:19,479 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13f50260f7a437ef753d61fda36692fe: 2024-11-07T12:41:19,479 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., storeName=13f50260f7a437ef753d61fda36692fe/C, priority=13, startTime=1730983278840; duration=0sec 2024-11-07T12:41:19,479 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-07T12:41:19,479 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13f50260f7a437ef753d61fda36692fe:C 2024-11-07T12:41:19,479 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13f50260f7a437ef753d61fda36692fe:C 2024-11-07T12:41:19,479 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:41:19,483 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13f50260f7a437ef753d61fda36692fe#A#compaction#60 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:41:19,484 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/1971acb7c50648a5b8d0467c2b083c56 is 50, key is test_row_0/A:col10/1730983279197/Put/seqid=0 2024-11-07T12:41:19,485 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:41:19,485 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 13f50260f7a437ef753d61fda36692fe/B is initiating minor compaction (all files) 2024-11-07T12:41:19,485 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13f50260f7a437ef753d61fda36692fe/B in TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:19,485 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/e044e1b4dc6244d986037942bd6b684b, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/2a4b1e6147f34934827d47d133a5ad5c, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/d06109e82b9f4f25badbfd68bd33ae0c] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp, totalSize=36.6 K 2024-11-07T12:41:19,487 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting e044e1b4dc6244d986037942bd6b684b, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1730983277838 2024-11-07T12:41:19,487 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a4b1e6147f34934827d47d133a5ad5c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1730983278212 2024-11-07T12:41:19,488 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting d06109e82b9f4f25badbfd68bd33ae0c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1730983279073 2024-11-07T12:41:19,504 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13f50260f7a437ef753d61fda36692fe#B#compaction#61 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:41:19,505 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/1f7e2729ac3649b4836caaa7cf4671b5 is 50, key is test_row_0/B:col10/1730983279197/Put/seqid=0 2024-11-07T12:41:19,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741897_1073 (size=12983) 2024-11-07T12:41:19,562 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/1971acb7c50648a5b8d0467c2b083c56 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/1971acb7c50648a5b8d0467c2b083c56 2024-11-07T12:41:19,598 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13f50260f7a437ef753d61fda36692fe/A of 13f50260f7a437ef753d61fda36692fe into 1971acb7c50648a5b8d0467c2b083c56(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:41:19,598 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13f50260f7a437ef753d61fda36692fe: 2024-11-07T12:41:19,599 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., storeName=13f50260f7a437ef753d61fda36692fe/A, priority=13, startTime=1730983279456; duration=0sec 2024-11-07T12:41:19,599 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:41:19,599 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13f50260f7a437ef753d61fda36692fe:A 2024-11-07T12:41:19,599 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:41:19,601 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:41:19,601 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): 13f50260f7a437ef753d61fda36692fe/C is initiating minor compaction (all files) 2024-11-07T12:41:19,601 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13f50260f7a437ef753d61fda36692fe/C in TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:19,601 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/5664d20731f7454fb1a564e7129d21a0, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/85083acf838440a1898c59406e57ee83, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/e206dd985be74ba3a7c8f4c49b0d7227] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp, totalSize=36.6 K 2024-11-07T12:41:19,602 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5664d20731f7454fb1a564e7129d21a0, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1730983277838 2024-11-07T12:41:19,602 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85083acf838440a1898c59406e57ee83, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1730983278212 2024-11-07T12:41:19,603 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting e206dd985be74ba3a7c8f4c49b0d7227, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1730983279073 2024-11-07T12:41:19,610 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 13f50260f7a437ef753d61fda36692fe 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-07T12:41:19,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 13f50260f7a437ef753d61fda36692fe 2024-11-07T12:41:19,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13f50260f7a437ef753d61fda36692fe, store=A 2024-11-07T12:41:19,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:19,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13f50260f7a437ef753d61fda36692fe, store=B 2024-11-07T12:41:19,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:19,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13f50260f7a437ef753d61fda36692fe, store=C 2024-11-07T12:41:19,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:19,652 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13f50260f7a437ef753d61fda36692fe#C#compaction#62 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:41:19,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741898_1074 (size=12983) 2024-11-07T12:41:19,656 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/10d62833c17c444b9ae2488ab66b306d is 50, key is test_row_0/C:col10/1730983279197/Put/seqid=0 2024-11-07T12:41:19,661 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/1f7e2729ac3649b4836caaa7cf4671b5 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/1f7e2729ac3649b4836caaa7cf4671b5 2024-11-07T12:41:19,666 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/ebe59b1fb0a344c7a42c314de5c43776 is 50, key is test_row_0/A:col10/1730983279589/Put/seqid=0 2024-11-07T12:41:19,675 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13f50260f7a437ef753d61fda36692fe/B of 13f50260f7a437ef753d61fda36692fe into 1f7e2729ac3649b4836caaa7cf4671b5(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:41:19,675 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13f50260f7a437ef753d61fda36692fe: 2024-11-07T12:41:19,675 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., storeName=13f50260f7a437ef753d61fda36692fe/B, priority=13, startTime=1730983279456; duration=0sec 2024-11-07T12:41:19,675 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:41:19,676 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13f50260f7a437ef753d61fda36692fe:B 2024-11-07T12:41:19,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741899_1075 (size=12983) 2024-11-07T12:41:19,731 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/10d62833c17c444b9ae2488ab66b306d as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/10d62833c17c444b9ae2488ab66b306d 2024-11-07T12:41:19,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741900_1076 (size=12301) 2024-11-07T12:41:19,742 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/ebe59b1fb0a344c7a42c314de5c43776 2024-11-07T12:41:19,745 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13f50260f7a437ef753d61fda36692fe/C of 13f50260f7a437ef753d61fda36692fe into 10d62833c17c444b9ae2488ab66b306d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:41:19,746 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13f50260f7a437ef753d61fda36692fe: 2024-11-07T12:41:19,746 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., storeName=13f50260f7a437ef753d61fda36692fe/C, priority=13, startTime=1730983279456; duration=0sec 2024-11-07T12:41:19,746 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:41:19,746 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13f50260f7a437ef753d61fda36692fe:C 2024-11-07T12:41:19,763 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/68c99c0d4c044e14b8ceb30d52102cc1 is 50, key is test_row_0/B:col10/1730983279589/Put/seqid=0 2024-11-07T12:41:19,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:19,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33346 deadline: 1730983339768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:19,785 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:19,785 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:19,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:19,785 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:19,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33370 deadline: 1730983339771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:19,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:19,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:19,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:19,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:19,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33340 deadline: 1730983339774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:19,787 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:19,787 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:19,787 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:19,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741901_1077 (size=12301) 2024-11-07T12:41:19,812 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/68c99c0d4c044e14b8ceb30d52102cc1 2024-11-07T12:41:19,846 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/005904b14b404d1cbcda5544c69d6bd3 is 50, key is test_row_0/C:col10/1730983279589/Put/seqid=0 2024-11-07T12:41:19,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:19,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33346 deadline: 1730983339892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:19,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741902_1078 (size=12301) 2024-11-07T12:41:19,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:19,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33340 deadline: 1730983339892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:19,902 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:19,902 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:19,902 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:19,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:19,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:19,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:19,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:19,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33370 deadline: 1730983339899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:19,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:19,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:19,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:20,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-07T12:41:20,042 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:41:20,043 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:41:20,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-07T12:41:20,046 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:41:20,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-07T12:41:20,047 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:41:20,047 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:41:20,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:20,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33346 deadline: 1730983340112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:20,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:20,114 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:20,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33370 deadline: 1730983340112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:20,114 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:20,114 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:20,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:20,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:20,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:20,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:20,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33340 deadline: 1730983340112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:20,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:20,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:20,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:20,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-07T12:41:20,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:20,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33334 deadline: 1730983340162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:20,163 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to db9ad1cb6cf9,34293,1730983260217 for 'test_row_0' in 13f50260f7a437ef753d61fda36692fe of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4265 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:41:20,164 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:20,164 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:20,164 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:20,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:20,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33368 deadline: 1730983340171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:20,178 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to db9ad1cb6cf9,34293,1730983260217 for 'test_row_0' in 13f50260f7a437ef753d61fda36692fe of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4280 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:41:20,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:20,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:20,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:20,209 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-07T12:41:20,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:20,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. as already flushing 2024-11-07T12:41:20,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:20,210 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:20,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:20,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:20,306 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/005904b14b404d1cbcda5544c69d6bd3 2024-11-07T12:41:20,341 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/ebe59b1fb0a344c7a42c314de5c43776 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/ebe59b1fb0a344c7a42c314de5c43776 2024-11-07T12:41:20,352 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/ebe59b1fb0a344c7a42c314de5c43776, entries=150, sequenceid=392, filesize=12.0 K 2024-11-07T12:41:20,354 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/68c99c0d4c044e14b8ceb30d52102cc1 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/68c99c0d4c044e14b8ceb30d52102cc1 2024-11-07T12:41:20,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-07T12:41:20,365 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-07T12:41:20,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:20,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. as already flushing 2024-11-07T12:41:20,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:20,366 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:20,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:20,374 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/68c99c0d4c044e14b8ceb30d52102cc1, entries=150, sequenceid=392, filesize=12.0 K 2024-11-07T12:41:20,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:20,385 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/005904b14b404d1cbcda5544c69d6bd3 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/005904b14b404d1cbcda5544c69d6bd3 2024-11-07T12:41:20,406 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/005904b14b404d1cbcda5544c69d6bd3, entries=150, sequenceid=392, filesize=12.0 K 2024-11-07T12:41:20,407 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 13f50260f7a437ef753d61fda36692fe in 797ms, sequenceid=392, compaction requested=false 2024-11-07T12:41:20,407 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 13f50260f7a437ef753d61fda36692fe: 2024-11-07T12:41:20,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 13f50260f7a437ef753d61fda36692fe 2024-11-07T12:41:20,430 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 13f50260f7a437ef753d61fda36692fe 3/3 column families, dataSize=114.05 KB heapSize=340.13 KB 2024-11-07T12:41:20,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13f50260f7a437ef753d61fda36692fe, store=A 2024-11-07T12:41:20,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:41:20,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13f50260f7a437ef753d61fda36692fe, store=B 2024-11-07T12:41:20,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:41:20,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13f50260f7a437ef753d61fda36692fe, store=C 2024-11-07T12:41:20,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:41:20,443 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/dff078fffd4b4516ad7eb566d73b262a is 50, key is test_row_0/A:col10/1730983280429/Put/seqid=0 2024-11-07T12:41:20,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741903_1079 (size=17181) 2024-11-07T12:41:20,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:20,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33346 deadline: 1730983340492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:20,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:20,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33340 deadline: 1730983340495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:20,505 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:20,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:20,505 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:20,505 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:20,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33370 deadline: 1730983340497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:20,505 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:20,505 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:20,505 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:20,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:20,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:20,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:20,537 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-07T12:41:20,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:20,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. as already flushing 2024-11-07T12:41:20,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:20,537 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:20,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:20,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:20,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:20,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33340 deadline: 1730983340612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:20,619 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:20,619 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:20,619 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:20,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:20,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33370 deadline: 1730983340612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:20,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:20,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33346 deadline: 1730983340613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:20,621 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:20,621 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:20,621 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:20,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:20,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:20,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:20,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-07T12:41:20,699 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-07T12:41:20,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:20,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. as already flushing 2024-11-07T12:41:20,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:20,700 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:20,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:20,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:20,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:20,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33346 deadline: 1730983340832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:20,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:20,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33370 deadline: 1730983340832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:20,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:20,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33340 deadline: 1730983340833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:20,844 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:20,844 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:20,844 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:20,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:20,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:20,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:20,845 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:20,845 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:20,845 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:20,861 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-07T12:41:20,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:20,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. as already flushing 2024-11-07T12:41:20,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:20,862 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:20,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:20,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:20,896 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/dff078fffd4b4516ad7eb566d73b262a 2024-11-07T12:41:20,955 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/13a958535761495fba745a35048e622d is 50, key is test_row_0/B:col10/1730983280429/Put/seqid=0 2024-11-07T12:41:21,027 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-07T12:41:21,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:21,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. as already flushing 2024-11-07T12:41:21,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:21,027 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:21,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:21,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741904_1080 (size=12301) 2024-11-07T12:41:21,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:21,044 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/13a958535761495fba745a35048e622d 2024-11-07T12:41:21,099 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/1ade7d26d27a4ab2b32aeda9ffa752eb is 50, key is test_row_0/C:col10/1730983280429/Put/seqid=0 2024-11-07T12:41:21,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741905_1081 (size=12301) 2024-11-07T12:41:21,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:21,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33370 deadline: 1730983341152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:21,156 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:21,156 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:21,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:21,156 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:21,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33346 deadline: 1730983341152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:21,156 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:21,156 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:21,156 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:21,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:21,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33340 deadline: 1730983341153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:21,159 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:21,159 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:21,159 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:21,190 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-07T12:41:21,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:21,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. as already flushing 2024-11-07T12:41:21,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:21,191 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:21,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:21,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-07T12:41:21,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:21,350 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-07T12:41:21,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:21,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. as already flushing 2024-11-07T12:41:21,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:21,350 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:21,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:21,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:21,505 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-07T12:41:21,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:21,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. as already flushing 2024-11-07T12:41:21,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:21,505 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:21,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:21,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:21,516 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/1ade7d26d27a4ab2b32aeda9ffa752eb 2024-11-07T12:41:21,548 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/dff078fffd4b4516ad7eb566d73b262a as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/dff078fffd4b4516ad7eb566d73b262a 2024-11-07T12:41:21,556 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/dff078fffd4b4516ad7eb566d73b262a, entries=250, sequenceid=415, filesize=16.8 K 2024-11-07T12:41:21,558 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/13a958535761495fba745a35048e622d as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/13a958535761495fba745a35048e622d 2024-11-07T12:41:21,564 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/13a958535761495fba745a35048e622d, entries=150, sequenceid=415, filesize=12.0 K 2024-11-07T12:41:21,565 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/1ade7d26d27a4ab2b32aeda9ffa752eb as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/1ade7d26d27a4ab2b32aeda9ffa752eb 2024-11-07T12:41:21,576 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/1ade7d26d27a4ab2b32aeda9ffa752eb, entries=150, sequenceid=415, filesize=12.0 K 2024-11-07T12:41:21,577 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 13f50260f7a437ef753d61fda36692fe in 1147ms, sequenceid=415, compaction requested=true 2024-11-07T12:41:21,577 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 13f50260f7a437ef753d61fda36692fe: 2024-11-07T12:41:21,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13f50260f7a437ef753d61fda36692fe:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:41:21,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:41:21,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13f50260f7a437ef753d61fda36692fe:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:41:21,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-07T12:41:21,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13f50260f7a437ef753d61fda36692fe:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:41:21,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-07T12:41:21,578 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:41:21,578 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:41:21,581 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:41:21,581 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 13f50260f7a437ef753d61fda36692fe/C is initiating minor compaction (all files) 2024-11-07T12:41:21,581 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42465 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:41:21,581 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13f50260f7a437ef753d61fda36692fe/C in TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:21,581 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): 13f50260f7a437ef753d61fda36692fe/A is initiating minor compaction (all files) 2024-11-07T12:41:21,581 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/10d62833c17c444b9ae2488ab66b306d, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/005904b14b404d1cbcda5544c69d6bd3, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/1ade7d26d27a4ab2b32aeda9ffa752eb] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp, totalSize=36.7 K 2024-11-07T12:41:21,584 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13f50260f7a437ef753d61fda36692fe/A in TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:21,584 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 10d62833c17c444b9ae2488ab66b306d, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1730983279073 2024-11-07T12:41:21,585 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/1971acb7c50648a5b8d0467c2b083c56, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/ebe59b1fb0a344c7a42c314de5c43776, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/dff078fffd4b4516ad7eb566d73b262a] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp, totalSize=41.5 K 2024-11-07T12:41:21,585 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1971acb7c50648a5b8d0467c2b083c56, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1730983279073 2024-11-07T12:41:21,585 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 005904b14b404d1cbcda5544c69d6bd3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1730983279235 2024-11-07T12:41:21,586 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting ebe59b1fb0a344c7a42c314de5c43776, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1730983279235 2024-11-07T12:41:21,586 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ade7d26d27a4ab2b32aeda9ffa752eb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1730983279725 2024-11-07T12:41:21,586 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting dff078fffd4b4516ad7eb566d73b262a, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1730983279725 2024-11-07T12:41:21,603 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13f50260f7a437ef753d61fda36692fe#C#compaction#69 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:41:21,604 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/5b2161d46a0d42cba6702a265f087613 is 50, key is test_row_0/C:col10/1730983280429/Put/seqid=0 2024-11-07T12:41:21,608 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13f50260f7a437ef753d61fda36692fe#A#compaction#70 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:41:21,609 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/3bffbfbb9be54b58ba9d395fe4794049 is 50, key is test_row_0/A:col10/1730983280429/Put/seqid=0 2024-11-07T12:41:21,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741906_1082 (size=13085) 2024-11-07T12:41:21,667 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 13f50260f7a437ef753d61fda36692fe 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-07T12:41:21,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 13f50260f7a437ef753d61fda36692fe 2024-11-07T12:41:21,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13f50260f7a437ef753d61fda36692fe, store=A 2024-11-07T12:41:21,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:21,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13f50260f7a437ef753d61fda36692fe, store=B 2024-11-07T12:41:21,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:21,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13f50260f7a437ef753d61fda36692fe, store=C 2024-11-07T12:41:21,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:21,670 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-07T12:41:21,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:21,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. as already flushing 2024-11-07T12:41:21,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:21,671 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:21,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:21,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:21,675 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/3bffbfbb9be54b58ba9d395fe4794049 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/3bffbfbb9be54b58ba9d395fe4794049 2024-11-07T12:41:21,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741907_1083 (size=13085) 2024-11-07T12:41:21,687 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/f6c9763666c945a9b62037afdf544625 is 50, key is test_row_0/A:col10/1730983281665/Put/seqid=0 2024-11-07T12:41:21,696 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13f50260f7a437ef753d61fda36692fe/A of 13f50260f7a437ef753d61fda36692fe into 3bffbfbb9be54b58ba9d395fe4794049(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:41:21,696 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13f50260f7a437ef753d61fda36692fe: 2024-11-07T12:41:21,696 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., storeName=13f50260f7a437ef753d61fda36692fe/A, priority=13, startTime=1730983281578; duration=0sec 2024-11-07T12:41:21,696 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:41:21,696 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13f50260f7a437ef753d61fda36692fe:A 2024-11-07T12:41:21,697 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:41:21,701 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:41:21,701 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): 13f50260f7a437ef753d61fda36692fe/B is initiating minor compaction (all files) 2024-11-07T12:41:21,701 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13f50260f7a437ef753d61fda36692fe/B in TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:21,702 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/1f7e2729ac3649b4836caaa7cf4671b5, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/68c99c0d4c044e14b8ceb30d52102cc1, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/13a958535761495fba745a35048e622d] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp, totalSize=36.7 K 2024-11-07T12:41:21,702 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/5b2161d46a0d42cba6702a265f087613 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/5b2161d46a0d42cba6702a265f087613 2024-11-07T12:41:21,702 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f7e2729ac3649b4836caaa7cf4671b5, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1730983279073 2024-11-07T12:41:21,703 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68c99c0d4c044e14b8ceb30d52102cc1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1730983279235 2024-11-07T12:41:21,704 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13a958535761495fba745a35048e622d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1730983279725 2024-11-07T12:41:21,715 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13f50260f7a437ef753d61fda36692fe/C of 13f50260f7a437ef753d61fda36692fe into 5b2161d46a0d42cba6702a265f087613(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:41:21,715 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13f50260f7a437ef753d61fda36692fe: 2024-11-07T12:41:21,715 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., storeName=13f50260f7a437ef753d61fda36692fe/C, priority=13, startTime=1730983281578; duration=0sec 2024-11-07T12:41:21,716 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:41:21,716 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13f50260f7a437ef753d61fda36692fe:C 2024-11-07T12:41:21,718 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13f50260f7a437ef753d61fda36692fe#B#compaction#72 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:41:21,719 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/7ef44fab22df4731b23ed0eaf48bb939 is 50, key is test_row_0/B:col10/1730983280429/Put/seqid=0 2024-11-07T12:41:21,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741908_1084 (size=17181) 2024-11-07T12:41:21,735 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/f6c9763666c945a9b62037afdf544625 2024-11-07T12:41:21,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:21,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33370 deadline: 1730983341746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:21,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:21,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:21,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:21,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:21,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33340 deadline: 1730983341746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:21,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:21,756 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:21,756 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:21,756 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:21,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33346 deadline: 1730983341748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:21,757 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:21,757 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:21,757 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:21,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741909_1085 (size=13085) 2024-11-07T12:41:21,776 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/7e3392a132d4433880dcd3921b31f12d is 50, key is test_row_0/B:col10/1730983281665/Put/seqid=0 2024-11-07T12:41:21,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741910_1086 (size=12301) 2024-11-07T12:41:21,821 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/7e3392a132d4433880dcd3921b31f12d 2024-11-07T12:41:21,834 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-07T12:41:21,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:21,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. as already flushing 2024-11-07T12:41:21,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:21,835 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/aeb7327277e74c91b9b9dc139bf0a60f is 50, key is test_row_0/C:col10/1730983281665/Put/seqid=0 2024-11-07T12:41:21,835 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:21,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:21,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:21,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741911_1087 (size=12301) 2024-11-07T12:41:21,865 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/aeb7327277e74c91b9b9dc139bf0a60f 2024-11-07T12:41:21,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:21,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33370 deadline: 1730983341863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:21,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:21,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:21,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:21,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:21,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33346 deadline: 1730983341863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:21,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:21,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:21,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:21,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:21,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33340 deadline: 1730983341863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:21,869 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:21,869 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:21,869 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:21,874 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/f6c9763666c945a9b62037afdf544625 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/f6c9763666c945a9b62037afdf544625 2024-11-07T12:41:21,886 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/f6c9763666c945a9b62037afdf544625, entries=250, sequenceid=428, filesize=16.8 K 2024-11-07T12:41:21,888 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/7e3392a132d4433880dcd3921b31f12d as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/7e3392a132d4433880dcd3921b31f12d 2024-11-07T12:41:21,901 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/7e3392a132d4433880dcd3921b31f12d, entries=150, sequenceid=428, filesize=12.0 K 2024-11-07T12:41:21,909 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/aeb7327277e74c91b9b9dc139bf0a60f as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/aeb7327277e74c91b9b9dc139bf0a60f 2024-11-07T12:41:21,930 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/aeb7327277e74c91b9b9dc139bf0a60f, entries=150, sequenceid=428, filesize=12.0 K 2024-11-07T12:41:21,931 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 13f50260f7a437ef753d61fda36692fe in 264ms, sequenceid=428, compaction requested=false 2024-11-07T12:41:21,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 13f50260f7a437ef753d61fda36692fe: 2024-11-07T12:41:21,998 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-07T12:41:21,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:21,998 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 13f50260f7a437ef753d61fda36692fe 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-07T12:41:21,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13f50260f7a437ef753d61fda36692fe, store=A 2024-11-07T12:41:21,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:22,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13f50260f7a437ef753d61fda36692fe, store=B 2024-11-07T12:41:22,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:22,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13f50260f7a437ef753d61fda36692fe, store=C 2024-11-07T12:41:22,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:22,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/fb75de18d6494582a773ae66dbfc98f5 is 50, key is test_row_0/A:col10/1730983281745/Put/seqid=0 2024-11-07T12:41:22,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741912_1088 (size=12301) 2024-11-07T12:41:22,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 13f50260f7a437ef753d61fda36692fe 2024-11-07T12:41:22,094 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. as already flushing 2024-11-07T12:41:22,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:22,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33370 deadline: 1730983342116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:22,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:22,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33346 deadline: 1730983342124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:22,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:22,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:22,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:22,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:22,132 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:22,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33340 deadline: 1730983342125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:22,132 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:22,132 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:22,132 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:22,132 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:22,132 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:22,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-07T12:41:22,237 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/7ef44fab22df4731b23ed0eaf48bb939 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/7ef44fab22df4731b23ed0eaf48bb939 2024-11-07T12:41:22,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:22,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33370 deadline: 1730983342242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:22,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:22,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:22,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:22,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:22,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33346 deadline: 1730983342242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:22,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:22,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:22,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:22,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:22,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33340 deadline: 1730983342243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:22,251 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:22,251 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:22,251 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:22,273 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13f50260f7a437ef753d61fda36692fe/B of 13f50260f7a437ef753d61fda36692fe into 7ef44fab22df4731b23ed0eaf48bb939(size=12.8 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:41:22,273 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13f50260f7a437ef753d61fda36692fe: 2024-11-07T12:41:22,273 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., storeName=13f50260f7a437ef753d61fda36692fe/B, priority=13, startTime=1730983281578; duration=0sec 2024-11-07T12:41:22,273 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:41:22,273 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13f50260f7a437ef753d61fda36692fe:B 2024-11-07T12:41:22,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:22,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33370 deadline: 1730983342452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:22,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:22,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:22,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:22,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:22,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33346 deadline: 1730983342453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:22,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:22,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:22,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:22,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:22,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33340 deadline: 1730983342461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:22,464 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:22,465 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:22,465 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:22,480 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/fb75de18d6494582a773ae66dbfc98f5 2024-11-07T12:41:22,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/82a9694bcb1444a5b61e69be65f1e2f0 is 50, key is test_row_0/B:col10/1730983281745/Put/seqid=0 2024-11-07T12:41:22,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741913_1089 (size=12301) 2024-11-07T12:41:22,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:22,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33370 deadline: 1730983342762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:22,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:22,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:22,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:22,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:22,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33346 deadline: 1730983342772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:22,777 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:22,777 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:22,777 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:22,777 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:22,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33340 deadline: 1730983342772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:22,778 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:22,778 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:22,778 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:22,907 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/82a9694bcb1444a5b61e69be65f1e2f0 2024-11-07T12:41:22,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/8d67c3ba613341cb8ff5cce665f1f1e6 is 50, key is test_row_0/C:col10/1730983281745/Put/seqid=0 2024-11-07T12:41:22,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741914_1090 (size=12301) 2024-11-07T12:41:22,965 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/8d67c3ba613341cb8ff5cce665f1f1e6 2024-11-07T12:41:22,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/fb75de18d6494582a773ae66dbfc98f5 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/fb75de18d6494582a773ae66dbfc98f5 2024-11-07T12:41:23,005 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/fb75de18d6494582a773ae66dbfc98f5, entries=150, sequenceid=449, filesize=12.0 K 2024-11-07T12:41:23,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/82a9694bcb1444a5b61e69be65f1e2f0 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/82a9694bcb1444a5b61e69be65f1e2f0 2024-11-07T12:41:23,021 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/82a9694bcb1444a5b61e69be65f1e2f0, entries=150, sequenceid=449, filesize=12.0 K 2024-11-07T12:41:23,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/8d67c3ba613341cb8ff5cce665f1f1e6 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/8d67c3ba613341cb8ff5cce665f1f1e6 2024-11-07T12:41:23,038 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/8d67c3ba613341cb8ff5cce665f1f1e6, entries=150, sequenceid=449, filesize=12.0 K 2024-11-07T12:41:23,041 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 13f50260f7a437ef753d61fda36692fe in 1043ms, sequenceid=449, compaction requested=true 2024-11-07T12:41:23,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 13f50260f7a437ef753d61fda36692fe: 2024-11-07T12:41:23,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:23,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-07T12:41:23,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-07T12:41:23,053 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-07T12:41:23,053 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0020 sec 2024-11-07T12:41:23,056 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 3.0110 sec 2024-11-07T12:41:23,287 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 13f50260f7a437ef753d61fda36692fe 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-07T12:41:23,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13f50260f7a437ef753d61fda36692fe, store=A 2024-11-07T12:41:23,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:23,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13f50260f7a437ef753d61fda36692fe, store=B 2024-11-07T12:41:23,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:23,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13f50260f7a437ef753d61fda36692fe, store=C 2024-11-07T12:41:23,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:23,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 13f50260f7a437ef753d61fda36692fe 2024-11-07T12:41:23,294 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/b2f476f3573248b9a30b1561228c0d9c is 50, key is test_row_0/A:col10/1730983283285/Put/seqid=0 2024-11-07T12:41:23,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741915_1091 (size=17181) 2024-11-07T12:41:23,321 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=465 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/b2f476f3573248b9a30b1561228c0d9c 2024-11-07T12:41:23,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:23,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33346 deadline: 1730983343327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:23,333 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:23,333 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:23,333 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:23,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:23,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33370 deadline: 1730983343330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:23,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:23,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:23,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:23,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:23,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33340 deadline: 1730983343332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:23,345 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:23,345 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:23,345 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:23,351 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/6a77310ee17541308df8b9cd6778f85a is 50, key is test_row_0/B:col10/1730983283285/Put/seqid=0 2024-11-07T12:41:23,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741916_1092 (size=12301) 2024-11-07T12:41:23,357 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=465 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/6a77310ee17541308df8b9cd6778f85a 2024-11-07T12:41:23,390 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/46f204fb373f45f186ee8672c04593c3 is 50, key is test_row_0/C:col10/1730983283285/Put/seqid=0 2024-11-07T12:41:23,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741917_1093 (size=12301) 2024-11-07T12:41:23,421 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=465 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/46f204fb373f45f186ee8672c04593c3 2024-11-07T12:41:23,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/b2f476f3573248b9a30b1561228c0d9c as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/b2f476f3573248b9a30b1561228c0d9c 2024-11-07T12:41:23,441 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/b2f476f3573248b9a30b1561228c0d9c, entries=250, sequenceid=465, filesize=16.8 K 2024-11-07T12:41:23,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:23,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:23,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33346 deadline: 1730983343441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:23,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33370 deadline: 1730983343442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:23,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:23,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:23,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:23,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:23,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:23,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:23,446 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/6a77310ee17541308df8b9cd6778f85a as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/6a77310ee17541308df8b9cd6778f85a 2024-11-07T12:41:23,455 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/6a77310ee17541308df8b9cd6778f85a, entries=150, sequenceid=465, filesize=12.0 K 2024-11-07T12:41:23,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:23,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33340 deadline: 1730983343452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:23,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:23,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:23,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:23,457 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/46f204fb373f45f186ee8672c04593c3 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/46f204fb373f45f186ee8672c04593c3 2024-11-07T12:41:23,486 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/46f204fb373f45f186ee8672c04593c3, entries=150, sequenceid=465, filesize=12.0 K 2024-11-07T12:41:23,488 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 13f50260f7a437ef753d61fda36692fe in 200ms, sequenceid=465, compaction requested=true 2024-11-07T12:41:23,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 13f50260f7a437ef753d61fda36692fe: 2024-11-07T12:41:23,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13f50260f7a437ef753d61fda36692fe:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:41:23,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:41:23,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13f50260f7a437ef753d61fda36692fe:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:41:23,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-07T12:41:23,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13f50260f7a437ef753d61fda36692fe:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:41:23,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-07T12:41:23,495 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:41:23,496 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:41:23,497 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 59748 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:41:23,497 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): 13f50260f7a437ef753d61fda36692fe/A is initiating minor compaction (all files) 2024-11-07T12:41:23,497 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13f50260f7a437ef753d61fda36692fe/A in TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:23,498 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/3bffbfbb9be54b58ba9d395fe4794049, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/f6c9763666c945a9b62037afdf544625, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/fb75de18d6494582a773ae66dbfc98f5, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/b2f476f3573248b9a30b1561228c0d9c] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp, totalSize=58.3 K 2024-11-07T12:41:23,499 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3bffbfbb9be54b58ba9d395fe4794049, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1730983279725 2024-11-07T12:41:23,499 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:41:23,499 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 13f50260f7a437ef753d61fda36692fe/C is initiating minor compaction (all files) 2024-11-07T12:41:23,499 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13f50260f7a437ef753d61fda36692fe/C in TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:23,499 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/5b2161d46a0d42cba6702a265f087613, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/aeb7327277e74c91b9b9dc139bf0a60f, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/8d67c3ba613341cb8ff5cce665f1f1e6, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/46f204fb373f45f186ee8672c04593c3] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp, totalSize=48.8 K 2024-11-07T12:41:23,500 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6c9763666c945a9b62037afdf544625, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1730983280479 2024-11-07T12:41:23,500 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b2161d46a0d42cba6702a265f087613, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1730983279725 2024-11-07T12:41:23,500 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb75de18d6494582a773ae66dbfc98f5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1730983281743 2024-11-07T12:41:23,502 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting aeb7327277e74c91b9b9dc139bf0a60f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1730983280479 2024-11-07T12:41:23,502 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2f476f3573248b9a30b1561228c0d9c, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1730983282102 2024-11-07T12:41:23,502 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d67c3ba613341cb8ff5cce665f1f1e6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1730983281743 2024-11-07T12:41:23,503 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 46f204fb373f45f186ee8672c04593c3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1730983282113 2024-11-07T12:41:23,520 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13f50260f7a437ef753d61fda36692fe#C#compaction#81 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:41:23,521 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/6766bb408db445bba7accb08c24adb92 is 50, key is test_row_0/C:col10/1730983283285/Put/seqid=0 2024-11-07T12:41:23,522 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13f50260f7a437ef753d61fda36692fe#A#compaction#82 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:41:23,523 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/2240fc11679f428dbf1d0e7d50061cd1 is 50, key is test_row_0/A:col10/1730983283285/Put/seqid=0 2024-11-07T12:41:23,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741918_1094 (size=13221) 2024-11-07T12:41:23,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741919_1095 (size=13221) 2024-11-07T12:41:23,558 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/2240fc11679f428dbf1d0e7d50061cd1 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/2240fc11679f428dbf1d0e7d50061cd1 2024-11-07T12:41:23,567 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 13f50260f7a437ef753d61fda36692fe/A of 13f50260f7a437ef753d61fda36692fe into 2240fc11679f428dbf1d0e7d50061cd1(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:41:23,567 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13f50260f7a437ef753d61fda36692fe: 2024-11-07T12:41:23,567 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., storeName=13f50260f7a437ef753d61fda36692fe/A, priority=12, startTime=1730983283494; duration=0sec 2024-11-07T12:41:23,568 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:41:23,568 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13f50260f7a437ef753d61fda36692fe:A 2024-11-07T12:41:23,568 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:41:23,570 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:41:23,570 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): 13f50260f7a437ef753d61fda36692fe/B is initiating minor compaction (all files) 2024-11-07T12:41:23,570 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13f50260f7a437ef753d61fda36692fe/B in TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:23,570 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/7ef44fab22df4731b23ed0eaf48bb939, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/7e3392a132d4433880dcd3921b31f12d, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/82a9694bcb1444a5b61e69be65f1e2f0, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/6a77310ee17541308df8b9cd6778f85a] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp, totalSize=48.8 K 2024-11-07T12:41:23,572 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ef44fab22df4731b23ed0eaf48bb939, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1730983279725 2024-11-07T12:41:23,572 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e3392a132d4433880dcd3921b31f12d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1730983280479 2024-11-07T12:41:23,573 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82a9694bcb1444a5b61e69be65f1e2f0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1730983281743 2024-11-07T12:41:23,573 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a77310ee17541308df8b9cd6778f85a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1730983282113 2024-11-07T12:41:23,606 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13f50260f7a437ef753d61fda36692fe#B#compaction#83 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:41:23,606 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/0f41adf36fc54bd99228e7e399a51300 is 50, key is test_row_0/B:col10/1730983283285/Put/seqid=0 2024-11-07T12:41:23,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741920_1096 (size=13221) 2024-11-07T12:41:23,661 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 13f50260f7a437ef753d61fda36692fe 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-07T12:41:23,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13f50260f7a437ef753d61fda36692fe, store=A 2024-11-07T12:41:23,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:23,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13f50260f7a437ef753d61fda36692fe, store=B 2024-11-07T12:41:23,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:23,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13f50260f7a437ef753d61fda36692fe, store=C 2024-11-07T12:41:23,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:23,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 13f50260f7a437ef753d61fda36692fe 2024-11-07T12:41:23,679 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/b60601ae668f43eba8a2f03aaf3fcba7 is 50, key is test_row_0/A:col10/1730983283319/Put/seqid=0 2024-11-07T12:41:23,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:23,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33340 deadline: 1730983343710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:23,713 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:23,713 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:23,713 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:23,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:23,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33346 deadline: 1730983343712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:23,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:23,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:23,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:23,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:23,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33370 deadline: 1730983343713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:23,722 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:23,722 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:23,722 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:23,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741921_1097 (size=17181) 2024-11-07T12:41:23,735 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=485 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/b60601ae668f43eba8a2f03aaf3fcba7 2024-11-07T12:41:23,756 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/5e30bb9adb134b53b72d49d63df68e6c is 50, key is test_row_0/B:col10/1730983283319/Put/seqid=0 2024-11-07T12:41:23,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741922_1098 (size=12301) 2024-11-07T12:41:23,823 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:23,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33340 deadline: 1730983343821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:23,824 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:23,825 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:23,825 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:23,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:23,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33346 deadline: 1730983343832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:23,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:23,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:23,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:23,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:23,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33370 deadline: 1730983343832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:23,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:23,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:23,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:23,987 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/6766bb408db445bba7accb08c24adb92 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/6766bb408db445bba7accb08c24adb92 2024-11-07T12:41:23,999 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 13f50260f7a437ef753d61fda36692fe/C of 13f50260f7a437ef753d61fda36692fe into 6766bb408db445bba7accb08c24adb92(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:41:23,999 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13f50260f7a437ef753d61fda36692fe: 2024-11-07T12:41:23,999 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., storeName=13f50260f7a437ef753d61fda36692fe/C, priority=12, startTime=1730983283495; duration=0sec 2024-11-07T12:41:23,999 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:41:23,999 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13f50260f7a437ef753d61fda36692fe:C 2024-11-07T12:41:24,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:24,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33340 deadline: 1730983344032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:24,036 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:24,036 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:24,036 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:24,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:24,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33346 deadline: 1730983344042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:24,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:24,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:24,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:24,050 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/0f41adf36fc54bd99228e7e399a51300 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/0f41adf36fc54bd99228e7e399a51300 2024-11-07T12:41:24,059 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 13f50260f7a437ef753d61fda36692fe/B of 13f50260f7a437ef753d61fda36692fe into 0f41adf36fc54bd99228e7e399a51300(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:41:24,059 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13f50260f7a437ef753d61fda36692fe: 2024-11-07T12:41:24,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:24,059 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., storeName=13f50260f7a437ef753d61fda36692fe/B, priority=12, startTime=1730983283495; duration=0sec 2024-11-07T12:41:24,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33370 deadline: 1730983344056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:24,059 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:41:24,059 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13f50260f7a437ef753d61fda36692fe:B 2024-11-07T12:41:24,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:24,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:24,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:24,187 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=485 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/5e30bb9adb134b53b72d49d63df68e6c 2024-11-07T12:41:24,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:24,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33334 deadline: 1730983344192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:24,199 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to db9ad1cb6cf9,34293,1730983260217 for 'test_row_0' in 13f50260f7a437ef753d61fda36692fe of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8300 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:41:24,200 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:24,200 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:24,200 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:24,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:24,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33368 deadline: 1730983344202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:24,204 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to db9ad1cb6cf9,34293,1730983260217 for 'test_row_0' in 13f50260f7a437ef753d61fda36692fe of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8306 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:41:24,205 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:24,205 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:24,205 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:24,217 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/b683e066875c48e89d910c7ce74c6bd0 is 50, key is test_row_0/C:col10/1730983283319/Put/seqid=0 2024-11-07T12:41:24,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-07T12:41:24,223 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:41:24,223 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:41:24,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-07T12:41:24,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-07T12:41:24,227 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:41:24,228 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:41:24,228 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:41:24,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741923_1099 (size=12301) 2024-11-07T12:41:24,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-07T12:41:24,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:24,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33340 deadline: 1730983344343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:24,345 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:24,345 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:24,345 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:24,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:24,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33346 deadline: 1730983344353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:24,356 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:24,357 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:24,357 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:24,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:24,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33370 deadline: 1730983344372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:24,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:24,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:24,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:24,386 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-07T12:41:24,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:24,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. as already flushing 2024-11-07T12:41:24,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:24,387 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:24,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:24,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:24,541 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-07T12:41:24,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:24,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. as already flushing 2024-11-07T12:41:24,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:24,542 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:24,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:24,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-07T12:41:24,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:24,669 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=485 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/b683e066875c48e89d910c7ce74c6bd0 2024-11-07T12:41:24,677 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/b60601ae668f43eba8a2f03aaf3fcba7 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/b60601ae668f43eba8a2f03aaf3fcba7 2024-11-07T12:41:24,688 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/b60601ae668f43eba8a2f03aaf3fcba7, entries=250, sequenceid=485, filesize=16.8 K 2024-11-07T12:41:24,690 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/5e30bb9adb134b53b72d49d63df68e6c as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/5e30bb9adb134b53b72d49d63df68e6c 2024-11-07T12:41:24,702 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/5e30bb9adb134b53b72d49d63df68e6c, entries=150, sequenceid=485, filesize=12.0 K 2024-11-07T12:41:24,702 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-07T12:41:24,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:24,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. as already flushing 2024-11-07T12:41:24,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:24,703 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:24,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:24,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:24,709 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/b683e066875c48e89d910c7ce74c6bd0 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/b683e066875c48e89d910c7ce74c6bd0 2024-11-07T12:41:24,719 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/b683e066875c48e89d910c7ce74c6bd0, entries=150, sequenceid=485, filesize=12.0 K 2024-11-07T12:41:24,725 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 13f50260f7a437ef753d61fda36692fe in 1064ms, sequenceid=485, compaction requested=false 2024-11-07T12:41:24,725 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 13f50260f7a437ef753d61fda36692fe: 2024-11-07T12:41:24,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-07T12:41:24,865 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-07T12:41:24,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:24,865 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 13f50260f7a437ef753d61fda36692fe 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-07T12:41:24,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13f50260f7a437ef753d61fda36692fe, store=A 2024-11-07T12:41:24,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:24,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13f50260f7a437ef753d61fda36692fe, store=B 2024-11-07T12:41:24,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:24,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13f50260f7a437ef753d61fda36692fe, store=C 2024-11-07T12:41:24,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:24,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 13f50260f7a437ef753d61fda36692fe 2024-11-07T12:41:24,872 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. as already flushing 2024-11-07T12:41:24,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/c9c8aa30bfd14d408441ecee8d882de5 is 50, key is test_row_0/A:col10/1730983283709/Put/seqid=0 2024-11-07T12:41:24,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741924_1100 (size=14741) 2024-11-07T12:41:24,922 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=502 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/c9c8aa30bfd14d408441ecee8d882de5 2024-11-07T12:41:24,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/53d680a4ec114dc3a8d703521c426426 is 50, key is test_row_0/B:col10/1730983283709/Put/seqid=0 2024-11-07T12:41:24,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:24,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33370 deadline: 1730983344956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:24,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:24,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:24,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:24,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:24,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33340 deadline: 1730983344960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:24,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:24,972 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:24,972 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:24,973 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:24,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33346 deadline: 1730983344962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:24,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:24,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:24,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:24,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741925_1101 (size=12301) 2024-11-07T12:41:25,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:25,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33340 deadline: 1730983345082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:25,090 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:25,090 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:25,090 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:25,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:25,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33346 deadline: 1730983345082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:25,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:25,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:25,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:25,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:25,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33370 deadline: 1730983345082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:25,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:25,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:25,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:25,272 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-07T12:41:25,273 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:41:25,274 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-07T12:41:25,274 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:41:25,277 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-07T12:41:25,278 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:41:25,279 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:41:25,280 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:41:25,280 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:41:25,280 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:41:25,280 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:41:25,280 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:41:25,280 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:41:25,280 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:41:25,281 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:41:25,282 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-07T12:41:25,282 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:41:25,282 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:41:25,282 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:41:25,282 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:41:25,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:25,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:25,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:25,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33370 deadline: 1730983345302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:25,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33346 deadline: 1730983345302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:25,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33340 deadline: 1730983345302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:25,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:25,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:25,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:25,303 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:25,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:25,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:25,303 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:25,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:25,303 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:41:25,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-07T12:41:25,404 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=502 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/53d680a4ec114dc3a8d703521c426426 2024-11-07T12:41:25,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/32e449659cc24d55ae5c05b564013012 is 50, key is test_row_0/C:col10/1730983283709/Put/seqid=0 2024-11-07T12:41:25,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741926_1102 (size=12301) 2024-11-07T12:41:25,452 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=502 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/32e449659cc24d55ae5c05b564013012 2024-11-07T12:41:25,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/c9c8aa30bfd14d408441ecee8d882de5 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/c9c8aa30bfd14d408441ecee8d882de5 2024-11-07T12:41:25,463 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/c9c8aa30bfd14d408441ecee8d882de5, entries=200, sequenceid=502, filesize=14.4 K 2024-11-07T12:41:25,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/53d680a4ec114dc3a8d703521c426426 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/53d680a4ec114dc3a8d703521c426426 2024-11-07T12:41:25,469 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/53d680a4ec114dc3a8d703521c426426, entries=150, sequenceid=502, filesize=12.0 K 2024-11-07T12:41:25,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/32e449659cc24d55ae5c05b564013012 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/32e449659cc24d55ae5c05b564013012 2024-11-07T12:41:25,477 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/32e449659cc24d55ae5c05b564013012, entries=150, sequenceid=502, filesize=12.0 K 2024-11-07T12:41:25,479 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 13f50260f7a437ef753d61fda36692fe in 614ms, sequenceid=502, compaction requested=true 2024-11-07T12:41:25,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 13f50260f7a437ef753d61fda36692fe: 2024-11-07T12:41:25,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:25,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-07T12:41:25,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-07T12:41:25,485 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-07T12:41:25,485 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2530 sec 2024-11-07T12:41:25,487 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.2630 sec 2024-11-07T12:41:25,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 13f50260f7a437ef753d61fda36692fe 2024-11-07T12:41:25,617 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 13f50260f7a437ef753d61fda36692fe 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-07T12:41:25,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13f50260f7a437ef753d61fda36692fe, store=A 2024-11-07T12:41:25,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:25,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13f50260f7a437ef753d61fda36692fe, store=B 2024-11-07T12:41:25,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:25,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13f50260f7a437ef753d61fda36692fe, store=C 2024-11-07T12:41:25,618 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-07T12:41:25,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:25,618 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:41:25,618 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:41:25,618 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:41:25,618 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:41:25,618 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-07T12:41:25,618 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:41:25,619 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:41:25,619 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:41:25,619 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-07T12:41:25,619 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:41:25,619 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:41:25,619 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:41:25,619 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:41:25,619 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:41:25,630 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/9827ac59d224468a9f73865e937ab39a is 50, key is test_row_0/A:col10/1730983285613/Put/seqid=0 2024-11-07T12:41:25,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741927_1103 (size=12301) 2024-11-07T12:41:25,639 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=522 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/9827ac59d224468a9f73865e937ab39a 2024-11-07T12:41:25,655 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/a8792e9d331a428e9272b389eb51dc12 is 50, key is test_row_0/B:col10/1730983285613/Put/seqid=0 2024-11-07T12:41:25,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741928_1104 (size=12301) 2024-11-07T12:41:26,064 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=522 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/a8792e9d331a428e9272b389eb51dc12 2024-11-07T12:41:26,081 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/81839378e7994d1b8d15d431cf863ec3 is 50, key is test_row_0/C:col10/1730983285613/Put/seqid=0 2024-11-07T12:41:26,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741929_1105 (size=12301) 2024-11-07T12:41:26,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-07T12:41:26,382 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:41:26,505 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=522 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/81839378e7994d1b8d15d431cf863ec3 2024-11-07T12:41:26,512 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/9827ac59d224468a9f73865e937ab39a as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/9827ac59d224468a9f73865e937ab39a 2024-11-07T12:41:26,520 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/9827ac59d224468a9f73865e937ab39a, entries=150, sequenceid=522, filesize=12.0 K 2024-11-07T12:41:26,523 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/a8792e9d331a428e9272b389eb51dc12 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/a8792e9d331a428e9272b389eb51dc12 2024-11-07T12:41:26,531 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/a8792e9d331a428e9272b389eb51dc12, entries=150, sequenceid=522, filesize=12.0 K 2024-11-07T12:41:26,532 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/81839378e7994d1b8d15d431cf863ec3 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/81839378e7994d1b8d15d431cf863ec3 2024-11-07T12:41:26,540 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/81839378e7994d1b8d15d431cf863ec3, entries=150, sequenceid=522, filesize=12.0 K 2024-11-07T12:41:26,541 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=0 B/0 for 13f50260f7a437ef753d61fda36692fe in 924ms, sequenceid=522, compaction requested=true 2024-11-07T12:41:26,541 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 13f50260f7a437ef753d61fda36692fe: 2024-11-07T12:41:26,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13f50260f7a437ef753d61fda36692fe:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:41:26,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:41:26,541 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:41:26,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13f50260f7a437ef753d61fda36692fe:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:41:26,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:41:26,542 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:41:26,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13f50260f7a437ef753d61fda36692fe:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:41:26,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:41:26,546 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57444 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:41:26,547 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): 13f50260f7a437ef753d61fda36692fe/A is initiating minor compaction (all files) 2024-11-07T12:41:26,547 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13f50260f7a437ef753d61fda36692fe/A in TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:26,547 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50124 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:41:26,547 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 13f50260f7a437ef753d61fda36692fe/B is initiating minor compaction (all files) 2024-11-07T12:41:26,547 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13f50260f7a437ef753d61fda36692fe/B in TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:26,548 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/2240fc11679f428dbf1d0e7d50061cd1, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/b60601ae668f43eba8a2f03aaf3fcba7, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/c9c8aa30bfd14d408441ecee8d882de5, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/9827ac59d224468a9f73865e937ab39a] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp, totalSize=56.1 K 2024-11-07T12:41:26,548 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/0f41adf36fc54bd99228e7e399a51300, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/5e30bb9adb134b53b72d49d63df68e6c, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/53d680a4ec114dc3a8d703521c426426, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/a8792e9d331a428e9272b389eb51dc12] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp, totalSize=48.9 K 2024-11-07T12:41:26,549 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f41adf36fc54bd99228e7e399a51300, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1730983282113 2024-11-07T12:41:26,549 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2240fc11679f428dbf1d0e7d50061cd1, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1730983282113 2024-11-07T12:41:26,549 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting b60601ae668f43eba8a2f03aaf3fcba7, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=485, earliestPutTs=1730983283319 2024-11-07T12:41:26,550 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e30bb9adb134b53b72d49d63df68e6c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=485, earliestPutTs=1730983283319 2024-11-07T12:41:26,550 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9c8aa30bfd14d408441ecee8d882de5, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=502, earliestPutTs=1730983283707 2024-11-07T12:41:26,550 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 53d680a4ec114dc3a8d703521c426426, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=502, earliestPutTs=1730983283707 2024-11-07T12:41:26,551 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9827ac59d224468a9f73865e937ab39a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=522, earliestPutTs=1730983284960 2024-11-07T12:41:26,551 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting a8792e9d331a428e9272b389eb51dc12, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=522, earliestPutTs=1730983284960 2024-11-07T12:41:26,580 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13f50260f7a437ef753d61fda36692fe#A#compaction#93 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:41:26,581 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/924be4e1ecfa4f6f9ae712ed2173478f is 50, key is test_row_0/A:col10/1730983285613/Put/seqid=0 2024-11-07T12:41:26,588 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13f50260f7a437ef753d61fda36692fe#B#compaction#94 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:41:26,588 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/3f2f0f23010a4ab9806523d5948ad83c is 50, key is test_row_0/B:col10/1730983285613/Put/seqid=0 2024-11-07T12:41:26,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741930_1106 (size=13357) 2024-11-07T12:41:26,635 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/924be4e1ecfa4f6f9ae712ed2173478f as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/924be4e1ecfa4f6f9ae712ed2173478f 2024-11-07T12:41:26,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741931_1107 (size=13357) 2024-11-07T12:41:26,649 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 13f50260f7a437ef753d61fda36692fe/A of 13f50260f7a437ef753d61fda36692fe into 924be4e1ecfa4f6f9ae712ed2173478f(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:41:26,649 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13f50260f7a437ef753d61fda36692fe: 2024-11-07T12:41:26,649 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., storeName=13f50260f7a437ef753d61fda36692fe/A, priority=12, startTime=1730983286541; duration=0sec 2024-11-07T12:41:26,649 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:41:26,649 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13f50260f7a437ef753d61fda36692fe:A 2024-11-07T12:41:26,649 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:41:26,651 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50124 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:41:26,652 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): 13f50260f7a437ef753d61fda36692fe/C is initiating minor compaction (all files) 2024-11-07T12:41:26,652 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13f50260f7a437ef753d61fda36692fe/C in TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:26,652 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/6766bb408db445bba7accb08c24adb92, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/b683e066875c48e89d910c7ce74c6bd0, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/32e449659cc24d55ae5c05b564013012, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/81839378e7994d1b8d15d431cf863ec3] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp, totalSize=48.9 K 2024-11-07T12:41:26,652 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6766bb408db445bba7accb08c24adb92, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1730983282113 2024-11-07T12:41:26,653 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting b683e066875c48e89d910c7ce74c6bd0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=485, earliestPutTs=1730983283319 2024-11-07T12:41:26,653 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32e449659cc24d55ae5c05b564013012, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=502, earliestPutTs=1730983283707 2024-11-07T12:41:26,654 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81839378e7994d1b8d15d431cf863ec3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=522, earliestPutTs=1730983284960 2024-11-07T12:41:26,676 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13f50260f7a437ef753d61fda36692fe#C#compaction#95 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:41:26,677 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/788a9361f1a84767996e29b71967c428 is 50, key is test_row_0/C:col10/1730983285613/Put/seqid=0 2024-11-07T12:41:26,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741932_1108 (size=13357) 2024-11-07T12:41:26,696 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/788a9361f1a84767996e29b71967c428 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/788a9361f1a84767996e29b71967c428 2024-11-07T12:41:26,708 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 13f50260f7a437ef753d61fda36692fe/C of 13f50260f7a437ef753d61fda36692fe into 788a9361f1a84767996e29b71967c428(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:41:26,708 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13f50260f7a437ef753d61fda36692fe: 2024-11-07T12:41:26,708 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., storeName=13f50260f7a437ef753d61fda36692fe/C, priority=12, startTime=1730983286542; duration=0sec 2024-11-07T12:41:26,708 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:41:26,708 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13f50260f7a437ef753d61fda36692fe:C 2024-11-07T12:41:27,064 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/3f2f0f23010a4ab9806523d5948ad83c as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/3f2f0f23010a4ab9806523d5948ad83c 2024-11-07T12:41:27,074 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 13f50260f7a437ef753d61fda36692fe/B of 13f50260f7a437ef753d61fda36692fe into 3f2f0f23010a4ab9806523d5948ad83c(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:41:27,076 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13f50260f7a437ef753d61fda36692fe: 2024-11-07T12:41:27,076 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., storeName=13f50260f7a437ef753d61fda36692fe/B, priority=12, startTime=1730983286541; duration=0sec 2024-11-07T12:41:27,076 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:41:27,079 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13f50260f7a437ef753d61fda36692fe:B 2024-11-07T12:41:27,836 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-07T12:41:34,217 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-07T12:41:34,217 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:41:34,217 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:41:34,217 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:41:34,217 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:41:34,264 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-07T12:41:34,264 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:41:34,264 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:41:34,264 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:41:34,264 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:41:34,264 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-07T12:41:34,265 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 122 2024-11-07T12:41:34,265 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 124 2024-11-07T12:41:34,265 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 104 2024-11-07T12:41:34,265 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 39 2024-11-07T12:41:34,265 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 41 2024-11-07T12:41:34,265 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-07T12:41:34,265 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1299 2024-11-07T12:41:34,265 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1293 2024-11-07T12:41:34,265 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-07T12:41:34,265 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 645 2024-11-07T12:41:34,265 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1935 rows 2024-11-07T12:41:34,265 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 650 2024-11-07T12:41:34,265 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1946 rows 2024-11-07T12:41:34,265 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-07T12:41:34,265 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T12:41:34,265 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:41:34,266 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:41:34,272 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:41:34,279 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-07T12:41:34,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-07T12:41:34,299 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730983294299"}]},"ts":"1730983294299"} 2024-11-07T12:41:34,302 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-07T12:41:34,302 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-07T12:41:34,304 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-07T12:41:34,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-07T12:41:34,311 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=13f50260f7a437ef753d61fda36692fe, UNASSIGN}] 2024-11-07T12:41:34,313 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=21, ppid=20, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=13f50260f7a437ef753d61fda36692fe, UNASSIGN 2024-11-07T12:41:34,315 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=13f50260f7a437ef753d61fda36692fe, regionState=CLOSING, regionLocation=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:34,318 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=13f50260f7a437ef753d61fda36692fe, UNASSIGN because future has completed 2024-11-07T12:41:34,320 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-07T12:41:34,321 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; CloseRegionProcedure 13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217}] 2024-11-07T12:41:34,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-07T12:41:34,486 INFO [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] handler.UnassignRegionHandler(122): Close 13f50260f7a437ef753d61fda36692fe 2024-11-07T12:41:34,487 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-07T12:41:34,487 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(1722): Closing 13f50260f7a437ef753d61fda36692fe, disabling compactions & flushes 2024-11-07T12:41:34,487 INFO [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:34,488 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:34,488 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. after waiting 0 ms 2024-11-07T12:41:34,488 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:34,488 INFO [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(2902): Flushing 13f50260f7a437ef753d61fda36692fe 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-11-07T12:41:34,488 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13f50260f7a437ef753d61fda36692fe, store=A 2024-11-07T12:41:34,488 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:34,488 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13f50260f7a437ef753d61fda36692fe, store=B 2024-11-07T12:41:34,488 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:34,488 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13f50260f7a437ef753d61fda36692fe, store=C 2024-11-07T12:41:34,489 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:34,499 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/2eec6d4f10b34fcc8329e777e1cd9c78 is 50, key is test_row_0/A:col10/1730983294262/Put/seqid=0 2024-11-07T12:41:34,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741933_1109 (size=7415) 2024-11-07T12:41:34,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-07T12:41:34,909 INFO [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=530 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/2eec6d4f10b34fcc8329e777e1cd9c78 2024-11-07T12:41:34,918 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/d63f47ef70b24757afd08ffef22bb8bd is 50, key is test_row_0/B:col10/1730983294262/Put/seqid=0 2024-11-07T12:41:34,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741934_1110 (size=7415) 2024-11-07T12:41:34,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-07T12:41:35,323 INFO [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=530 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/d63f47ef70b24757afd08ffef22bb8bd 2024-11-07T12:41:35,332 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/2e08dab1cba64346a5ae1af0026de971 is 50, key is test_row_0/C:col10/1730983294262/Put/seqid=0 2024-11-07T12:41:35,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741935_1111 (size=7415) 2024-11-07T12:41:35,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-07T12:41:35,737 INFO [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=530 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/2e08dab1cba64346a5ae1af0026de971 2024-11-07T12:41:35,743 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/A/2eec6d4f10b34fcc8329e777e1cd9c78 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/2eec6d4f10b34fcc8329e777e1cd9c78 2024-11-07T12:41:35,747 INFO [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/2eec6d4f10b34fcc8329e777e1cd9c78, entries=50, sequenceid=530, filesize=7.2 K 2024-11-07T12:41:35,748 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/B/d63f47ef70b24757afd08ffef22bb8bd as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/d63f47ef70b24757afd08ffef22bb8bd 2024-11-07T12:41:35,753 INFO [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/d63f47ef70b24757afd08ffef22bb8bd, entries=50, sequenceid=530, filesize=7.2 K 2024-11-07T12:41:35,754 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/.tmp/C/2e08dab1cba64346a5ae1af0026de971 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/2e08dab1cba64346a5ae1af0026de971 2024-11-07T12:41:35,759 INFO [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/2e08dab1cba64346a5ae1af0026de971, entries=50, sequenceid=530, filesize=7.2 K 2024-11-07T12:41:35,759 INFO [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 13f50260f7a437ef753d61fda36692fe in 1271ms, sequenceid=530, compaction requested=false 2024-11-07T12:41:35,760 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/887166ce52d64452878350f3f12ab91d, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/8c70e7b9422f45088d9e8282cde78313, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/984f6f600dd44af5a432476790bf6f1b, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/e6b2c5e388124330a264171732c981e3, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/cac0449fd52048d49b229bf1e1cae684, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/7ce98257f13c41c5841bedffb34c4ef5, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/6a2ec1ae72ca4cbfb08375698bb9e5e8, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/68784e5391194706a714e051045ee786, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/3ffc455d920d4467a2c74ed7a9e9da38, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/4d7e0508c5014a4a8c167717b99a6336, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/de614f860f6243329267ec3c07070732, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/e13948cacc374efc966cefd5e6dd7207, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/9f3166eb5b794decbdd50fdb9e6c0230, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/5c8d99db3f8149b88db0493a97149117, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/ba5ac0ad4d8a4542934813bc04ca6b5a, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/f910199589ed409794c0442f2583e62f, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/b0ededf8e5e24e5b86963104dfa8d905, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/0b95ee000f9e4094bd0925b6de91945d, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/12649c51fad447469a8f2912cf01e9b9, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/a0c29527ec0140c1be1f528b2bd81cb9, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/1971acb7c50648a5b8d0467c2b083c56, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/ebe59b1fb0a344c7a42c314de5c43776, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/dff078fffd4b4516ad7eb566d73b262a, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/3bffbfbb9be54b58ba9d395fe4794049, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/f6c9763666c945a9b62037afdf544625, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/fb75de18d6494582a773ae66dbfc98f5, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/b2f476f3573248b9a30b1561228c0d9c, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/2240fc11679f428dbf1d0e7d50061cd1, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/b60601ae668f43eba8a2f03aaf3fcba7, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/c9c8aa30bfd14d408441ecee8d882de5, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/9827ac59d224468a9f73865e937ab39a] to archive 2024-11-07T12:41:35,764 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T12:41:35,771 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/887166ce52d64452878350f3f12ab91d to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/887166ce52d64452878350f3f12ab91d 2024-11-07T12:41:35,773 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/8c70e7b9422f45088d9e8282cde78313 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/8c70e7b9422f45088d9e8282cde78313 2024-11-07T12:41:35,774 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/984f6f600dd44af5a432476790bf6f1b to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/984f6f600dd44af5a432476790bf6f1b 2024-11-07T12:41:35,775 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/e6b2c5e388124330a264171732c981e3 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/e6b2c5e388124330a264171732c981e3 2024-11-07T12:41:35,777 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/cac0449fd52048d49b229bf1e1cae684 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/cac0449fd52048d49b229bf1e1cae684 2024-11-07T12:41:35,778 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/7ce98257f13c41c5841bedffb34c4ef5 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/7ce98257f13c41c5841bedffb34c4ef5 2024-11-07T12:41:35,780 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/6a2ec1ae72ca4cbfb08375698bb9e5e8 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/6a2ec1ae72ca4cbfb08375698bb9e5e8 2024-11-07T12:41:35,781 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/68784e5391194706a714e051045ee786 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/68784e5391194706a714e051045ee786 2024-11-07T12:41:35,782 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/3ffc455d920d4467a2c74ed7a9e9da38 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/3ffc455d920d4467a2c74ed7a9e9da38 2024-11-07T12:41:35,783 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/4d7e0508c5014a4a8c167717b99a6336 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/4d7e0508c5014a4a8c167717b99a6336 2024-11-07T12:41:35,785 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/de614f860f6243329267ec3c07070732 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/de614f860f6243329267ec3c07070732 2024-11-07T12:41:35,786 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/e13948cacc374efc966cefd5e6dd7207 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/e13948cacc374efc966cefd5e6dd7207 2024-11-07T12:41:35,787 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/9f3166eb5b794decbdd50fdb9e6c0230 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/9f3166eb5b794decbdd50fdb9e6c0230 2024-11-07T12:41:35,789 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/5c8d99db3f8149b88db0493a97149117 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/5c8d99db3f8149b88db0493a97149117 2024-11-07T12:41:35,790 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/ba5ac0ad4d8a4542934813bc04ca6b5a to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/ba5ac0ad4d8a4542934813bc04ca6b5a 2024-11-07T12:41:35,791 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/f910199589ed409794c0442f2583e62f to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/f910199589ed409794c0442f2583e62f 2024-11-07T12:41:35,793 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/b0ededf8e5e24e5b86963104dfa8d905 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/b0ededf8e5e24e5b86963104dfa8d905 2024-11-07T12:41:35,794 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/0b95ee000f9e4094bd0925b6de91945d to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/0b95ee000f9e4094bd0925b6de91945d 2024-11-07T12:41:35,796 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/12649c51fad447469a8f2912cf01e9b9 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/12649c51fad447469a8f2912cf01e9b9 2024-11-07T12:41:35,797 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/a0c29527ec0140c1be1f528b2bd81cb9 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/a0c29527ec0140c1be1f528b2bd81cb9 2024-11-07T12:41:35,798 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/1971acb7c50648a5b8d0467c2b083c56 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/1971acb7c50648a5b8d0467c2b083c56 2024-11-07T12:41:35,800 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/ebe59b1fb0a344c7a42c314de5c43776 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/ebe59b1fb0a344c7a42c314de5c43776 2024-11-07T12:41:35,801 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/dff078fffd4b4516ad7eb566d73b262a to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/dff078fffd4b4516ad7eb566d73b262a 2024-11-07T12:41:35,802 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/3bffbfbb9be54b58ba9d395fe4794049 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/3bffbfbb9be54b58ba9d395fe4794049 2024-11-07T12:41:35,803 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/f6c9763666c945a9b62037afdf544625 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/f6c9763666c945a9b62037afdf544625 2024-11-07T12:41:35,805 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/fb75de18d6494582a773ae66dbfc98f5 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/fb75de18d6494582a773ae66dbfc98f5 2024-11-07T12:41:35,807 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/b2f476f3573248b9a30b1561228c0d9c to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/b2f476f3573248b9a30b1561228c0d9c 2024-11-07T12:41:35,808 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/2240fc11679f428dbf1d0e7d50061cd1 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/2240fc11679f428dbf1d0e7d50061cd1 2024-11-07T12:41:35,810 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/b60601ae668f43eba8a2f03aaf3fcba7 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/b60601ae668f43eba8a2f03aaf3fcba7 2024-11-07T12:41:35,812 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/c9c8aa30bfd14d408441ecee8d882de5 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/c9c8aa30bfd14d408441ecee8d882de5 2024-11-07T12:41:35,813 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/9827ac59d224468a9f73865e937ab39a to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/9827ac59d224468a9f73865e937ab39a 2024-11-07T12:41:35,831 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/debd4f8c1c994e9c9f32358e5cae6558, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/635e26c6edd74df4ac9b8674282b0522, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/c910fe4263ab42e3a28160e585f63210, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/f858673ce0cd4fee89de78ca69a0830f, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/18e5f240c667405fbfaf94be498c1e2e, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/67124a5d88fe4e69857c3d161412b606, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/2ab0eb7528cd40a0b1ad4741e4353362, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/5a37914d7b9d4902baee54a711860dd3, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/e78b648d4c224c56b73e4ee612eb63af, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/3e17a8db69e04f15bf88f3720ea288e9, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/b595992aea684f65ba822ea3c929a2a5, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/bc1a7d43d2784fefbdb460727bdffc41, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/c539739013124ba689aece438ba3006b, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/b9479869f2bc455fa3833a74b205f591, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/cb8ade718bc446c388991c12fa5d4dc1, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/a3674a1e469d466f9c3947511e7cf8d3, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/e044e1b4dc6244d986037942bd6b684b, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/b1d69d26b95d419c823ac2eb5fa2420c, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/2a4b1e6147f34934827d47d133a5ad5c, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/1f7e2729ac3649b4836caaa7cf4671b5, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/d06109e82b9f4f25badbfd68bd33ae0c, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/68c99c0d4c044e14b8ceb30d52102cc1, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/7ef44fab22df4731b23ed0eaf48bb939, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/13a958535761495fba745a35048e622d, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/7e3392a132d4433880dcd3921b31f12d, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/82a9694bcb1444a5b61e69be65f1e2f0, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/0f41adf36fc54bd99228e7e399a51300, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/6a77310ee17541308df8b9cd6778f85a, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/5e30bb9adb134b53b72d49d63df68e6c, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/53d680a4ec114dc3a8d703521c426426, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/a8792e9d331a428e9272b389eb51dc12] to archive 2024-11-07T12:41:35,833 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T12:41:35,836 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/debd4f8c1c994e9c9f32358e5cae6558 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/debd4f8c1c994e9c9f32358e5cae6558 2024-11-07T12:41:35,838 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/635e26c6edd74df4ac9b8674282b0522 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/635e26c6edd74df4ac9b8674282b0522 2024-11-07T12:41:35,845 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/c910fe4263ab42e3a28160e585f63210 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/c910fe4263ab42e3a28160e585f63210 2024-11-07T12:41:35,853 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/f858673ce0cd4fee89de78ca69a0830f to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/f858673ce0cd4fee89de78ca69a0830f 2024-11-07T12:41:35,856 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/18e5f240c667405fbfaf94be498c1e2e to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/18e5f240c667405fbfaf94be498c1e2e 2024-11-07T12:41:35,861 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/67124a5d88fe4e69857c3d161412b606 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/67124a5d88fe4e69857c3d161412b606 2024-11-07T12:41:35,862 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/2ab0eb7528cd40a0b1ad4741e4353362 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/2ab0eb7528cd40a0b1ad4741e4353362 2024-11-07T12:41:35,864 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/5a37914d7b9d4902baee54a711860dd3 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/5a37914d7b9d4902baee54a711860dd3 2024-11-07T12:41:35,866 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/e78b648d4c224c56b73e4ee612eb63af to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/e78b648d4c224c56b73e4ee612eb63af 2024-11-07T12:41:35,867 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/3e17a8db69e04f15bf88f3720ea288e9 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/3e17a8db69e04f15bf88f3720ea288e9 2024-11-07T12:41:35,869 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/b595992aea684f65ba822ea3c929a2a5 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/b595992aea684f65ba822ea3c929a2a5 2024-11-07T12:41:35,870 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/bc1a7d43d2784fefbdb460727bdffc41 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/bc1a7d43d2784fefbdb460727bdffc41 2024-11-07T12:41:35,871 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/c539739013124ba689aece438ba3006b to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/c539739013124ba689aece438ba3006b 2024-11-07T12:41:35,872 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/b9479869f2bc455fa3833a74b205f591 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/b9479869f2bc455fa3833a74b205f591 2024-11-07T12:41:35,873 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/cb8ade718bc446c388991c12fa5d4dc1 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/cb8ade718bc446c388991c12fa5d4dc1 2024-11-07T12:41:35,874 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/a3674a1e469d466f9c3947511e7cf8d3 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/a3674a1e469d466f9c3947511e7cf8d3 2024-11-07T12:41:35,876 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/e044e1b4dc6244d986037942bd6b684b to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/e044e1b4dc6244d986037942bd6b684b 2024-11-07T12:41:35,877 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/b1d69d26b95d419c823ac2eb5fa2420c to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/b1d69d26b95d419c823ac2eb5fa2420c 2024-11-07T12:41:35,878 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/2a4b1e6147f34934827d47d133a5ad5c to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/2a4b1e6147f34934827d47d133a5ad5c 2024-11-07T12:41:35,879 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/1f7e2729ac3649b4836caaa7cf4671b5 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/1f7e2729ac3649b4836caaa7cf4671b5 2024-11-07T12:41:35,880 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/d06109e82b9f4f25badbfd68bd33ae0c to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/d06109e82b9f4f25badbfd68bd33ae0c 2024-11-07T12:41:35,881 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/68c99c0d4c044e14b8ceb30d52102cc1 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/68c99c0d4c044e14b8ceb30d52102cc1 2024-11-07T12:41:35,883 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/7ef44fab22df4731b23ed0eaf48bb939 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/7ef44fab22df4731b23ed0eaf48bb939 2024-11-07T12:41:35,884 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/13a958535761495fba745a35048e622d to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/13a958535761495fba745a35048e622d 2024-11-07T12:41:35,885 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/7e3392a132d4433880dcd3921b31f12d to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/7e3392a132d4433880dcd3921b31f12d 2024-11-07T12:41:35,886 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/82a9694bcb1444a5b61e69be65f1e2f0 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/82a9694bcb1444a5b61e69be65f1e2f0 2024-11-07T12:41:35,887 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/0f41adf36fc54bd99228e7e399a51300 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/0f41adf36fc54bd99228e7e399a51300 2024-11-07T12:41:35,889 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/6a77310ee17541308df8b9cd6778f85a to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/6a77310ee17541308df8b9cd6778f85a 2024-11-07T12:41:35,890 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/5e30bb9adb134b53b72d49d63df68e6c to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/5e30bb9adb134b53b72d49d63df68e6c 2024-11-07T12:41:35,891 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/53d680a4ec114dc3a8d703521c426426 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/53d680a4ec114dc3a8d703521c426426 2024-11-07T12:41:35,892 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/a8792e9d331a428e9272b389eb51dc12 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/a8792e9d331a428e9272b389eb51dc12 2024-11-07T12:41:35,894 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/9f2c21c4acf44a4b84aa28cbb43485dd, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/f1bc8ccf47184853899915793b487071, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/5ebf3ab4dd57492ca2baff06cb7efd4d, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/8145a18ad4e0432c812ff4b68c2c71ae, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/99bcb8e4b7af48daa454930588fa90f9, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/45523d94041e41c09e3d2554757ee7d8, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/0f457255476e43daa9c9065a2bef4730, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/c2c96961925f423aa6f45edfa973167f, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/310e2a7ec25c41b99df1f6e06f8dff7c, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/bd4a3f5c2d754d499398190dd77aef9f, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/b3722eb306b7474f94a13cb5c7475358, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/9af50e3fcf4d4761bb6f701b1354e21a, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/684e250660ed4cb08608dc0b81c057b3, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/7501b5b83cbe4d4b9ed84d9c42d3b1ec, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/1b510c1251844884b3e845483d058bba, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/c8f233088adb46d99f0c78e90497b035, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/5664d20731f7454fb1a564e7129d21a0, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/95102599eaa74ca1b1a3d88f5b6ffc9e, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/85083acf838440a1898c59406e57ee83, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/10d62833c17c444b9ae2488ab66b306d, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/e206dd985be74ba3a7c8f4c49b0d7227, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/005904b14b404d1cbcda5544c69d6bd3, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/5b2161d46a0d42cba6702a265f087613, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/1ade7d26d27a4ab2b32aeda9ffa752eb, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/aeb7327277e74c91b9b9dc139bf0a60f, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/8d67c3ba613341cb8ff5cce665f1f1e6, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/6766bb408db445bba7accb08c24adb92, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/46f204fb373f45f186ee8672c04593c3, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/b683e066875c48e89d910c7ce74c6bd0, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/32e449659cc24d55ae5c05b564013012, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/81839378e7994d1b8d15d431cf863ec3] to archive 2024-11-07T12:41:35,895 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T12:41:35,897 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/9f2c21c4acf44a4b84aa28cbb43485dd to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/9f2c21c4acf44a4b84aa28cbb43485dd 2024-11-07T12:41:35,898 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/f1bc8ccf47184853899915793b487071 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/f1bc8ccf47184853899915793b487071 2024-11-07T12:41:35,899 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/5ebf3ab4dd57492ca2baff06cb7efd4d to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/5ebf3ab4dd57492ca2baff06cb7efd4d 2024-11-07T12:41:35,901 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/8145a18ad4e0432c812ff4b68c2c71ae to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/8145a18ad4e0432c812ff4b68c2c71ae 2024-11-07T12:41:35,902 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/99bcb8e4b7af48daa454930588fa90f9 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/99bcb8e4b7af48daa454930588fa90f9 2024-11-07T12:41:35,903 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/45523d94041e41c09e3d2554757ee7d8 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/45523d94041e41c09e3d2554757ee7d8 2024-11-07T12:41:35,904 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/0f457255476e43daa9c9065a2bef4730 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/0f457255476e43daa9c9065a2bef4730 2024-11-07T12:41:35,905 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/c2c96961925f423aa6f45edfa973167f to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/c2c96961925f423aa6f45edfa973167f 2024-11-07T12:41:35,906 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/310e2a7ec25c41b99df1f6e06f8dff7c to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/310e2a7ec25c41b99df1f6e06f8dff7c 2024-11-07T12:41:35,907 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/bd4a3f5c2d754d499398190dd77aef9f to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/bd4a3f5c2d754d499398190dd77aef9f 2024-11-07T12:41:35,908 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/b3722eb306b7474f94a13cb5c7475358 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/b3722eb306b7474f94a13cb5c7475358 2024-11-07T12:41:35,909 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/9af50e3fcf4d4761bb6f701b1354e21a to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/9af50e3fcf4d4761bb6f701b1354e21a 2024-11-07T12:41:35,910 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/684e250660ed4cb08608dc0b81c057b3 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/684e250660ed4cb08608dc0b81c057b3 2024-11-07T12:41:35,911 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/7501b5b83cbe4d4b9ed84d9c42d3b1ec to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/7501b5b83cbe4d4b9ed84d9c42d3b1ec 2024-11-07T12:41:35,912 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/1b510c1251844884b3e845483d058bba to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/1b510c1251844884b3e845483d058bba 2024-11-07T12:41:35,913 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/c8f233088adb46d99f0c78e90497b035 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/c8f233088adb46d99f0c78e90497b035 2024-11-07T12:41:35,914 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/5664d20731f7454fb1a564e7129d21a0 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/5664d20731f7454fb1a564e7129d21a0 2024-11-07T12:41:35,915 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/95102599eaa74ca1b1a3d88f5b6ffc9e to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/95102599eaa74ca1b1a3d88f5b6ffc9e 2024-11-07T12:41:35,916 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/85083acf838440a1898c59406e57ee83 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/85083acf838440a1898c59406e57ee83 2024-11-07T12:41:35,918 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/10d62833c17c444b9ae2488ab66b306d to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/10d62833c17c444b9ae2488ab66b306d 2024-11-07T12:41:35,919 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/e206dd985be74ba3a7c8f4c49b0d7227 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/e206dd985be74ba3a7c8f4c49b0d7227 2024-11-07T12:41:35,920 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/005904b14b404d1cbcda5544c69d6bd3 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/005904b14b404d1cbcda5544c69d6bd3 2024-11-07T12:41:35,921 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/5b2161d46a0d42cba6702a265f087613 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/5b2161d46a0d42cba6702a265f087613 2024-11-07T12:41:35,922 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/1ade7d26d27a4ab2b32aeda9ffa752eb to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/1ade7d26d27a4ab2b32aeda9ffa752eb 2024-11-07T12:41:35,923 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/aeb7327277e74c91b9b9dc139bf0a60f to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/aeb7327277e74c91b9b9dc139bf0a60f 2024-11-07T12:41:35,924 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/8d67c3ba613341cb8ff5cce665f1f1e6 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/8d67c3ba613341cb8ff5cce665f1f1e6 2024-11-07T12:41:35,925 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/6766bb408db445bba7accb08c24adb92 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/6766bb408db445bba7accb08c24adb92 2024-11-07T12:41:35,927 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/46f204fb373f45f186ee8672c04593c3 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/46f204fb373f45f186ee8672c04593c3 2024-11-07T12:41:35,928 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/b683e066875c48e89d910c7ce74c6bd0 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/b683e066875c48e89d910c7ce74c6bd0 2024-11-07T12:41:35,929 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/32e449659cc24d55ae5c05b564013012 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/32e449659cc24d55ae5c05b564013012 2024-11-07T12:41:35,930 DEBUG [StoreCloser-TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/81839378e7994d1b8d15d431cf863ec3 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/81839378e7994d1b8d15d431cf863ec3 2024-11-07T12:41:35,935 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/recovered.edits/533.seqid, newMaxSeqId=533, maxSeqId=1 2024-11-07T12:41:35,937 INFO [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. 2024-11-07T12:41:35,937 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(1676): Region close journal for 13f50260f7a437ef753d61fda36692fe: Waiting for close lock at 1730983294487Running coprocessor pre-close hooks at 1730983294487Disabling compacts and flushes for region at 1730983294487Disabling writes for close at 1730983294488 (+1 ms)Obtaining lock to block concurrent updates at 1730983294488Preparing flush snapshotting stores in 13f50260f7a437ef753d61fda36692fe at 1730983294488Finished memstore snapshotting TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=41520, getOffHeapSize=0, getCellsCount=300 at 1730983294489 (+1 ms)Flushing stores of TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe. at 1730983294493 (+4 ms)Flushing 13f50260f7a437ef753d61fda36692fe/A: creating writer at 1730983294493Flushing 13f50260f7a437ef753d61fda36692fe/A: appending metadata at 1730983294498 (+5 ms)Flushing 13f50260f7a437ef753d61fda36692fe/A: closing flushed file at 1730983294498Flushing 13f50260f7a437ef753d61fda36692fe/B: creating writer at 1730983294915 (+417 ms)Flushing 13f50260f7a437ef753d61fda36692fe/B: appending metadata at 1730983294918 (+3 ms)Flushing 13f50260f7a437ef753d61fda36692fe/B: closing flushed file at 1730983294918Flushing 13f50260f7a437ef753d61fda36692fe/C: creating writer at 1730983295329 (+411 ms)Flushing 13f50260f7a437ef753d61fda36692fe/C: appending metadata at 1730983295331 (+2 ms)Flushing 13f50260f7a437ef753d61fda36692fe/C: closing flushed file at 1730983295332 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1f6d573: reopening flushed file at 1730983295742 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@28ac9d18: reopening flushed file at 1730983295748 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@54617680: reopening flushed file at 1730983295753 (+5 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 13f50260f7a437ef753d61fda36692fe in 1271ms, sequenceid=530, compaction requested=false at 1730983295759 (+6 ms)Writing region close event to WAL at 1730983295932 (+173 ms)Running coprocessor post-close hooks at 1730983295937 (+5 ms)Closed at 1730983295937 2024-11-07T12:41:35,939 INFO [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] handler.UnassignRegionHandler(157): Closed 13f50260f7a437ef753d61fda36692fe 2024-11-07T12:41:35,940 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=13f50260f7a437ef753d61fda36692fe, regionState=CLOSED 2024-11-07T12:41:35,942 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=21, state=RUNNABLE, hasLock=false; CloseRegionProcedure 13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 because future has completed 2024-11-07T12:41:35,945 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-07T12:41:35,945 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; CloseRegionProcedure 13f50260f7a437ef753d61fda36692fe, server=db9ad1cb6cf9,34293,1730983260217 in 1.6220 sec 2024-11-07T12:41:35,947 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-11-07T12:41:35,947 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=13f50260f7a437ef753d61fda36692fe, UNASSIGN in 1.6340 sec 2024-11-07T12:41:35,949 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-07T12:41:35,949 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6440 sec 2024-11-07T12:41:35,951 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730983295951"}]},"ts":"1730983295951"} 2024-11-07T12:41:35,952 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-07T12:41:35,952 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-07T12:41:35,954 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6680 sec 2024-11-07T12:41:36,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-07T12:41:36,463 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:41:36,466 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-07T12:41:36,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:41:36,474 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=23, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:41:36,476 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=23, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:41:36,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-07T12:41:36,481 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe 2024-11-07T12:41:36,487 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A, FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B, FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C, FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/recovered.edits] 2024-11-07T12:41:36,494 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/2eec6d4f10b34fcc8329e777e1cd9c78 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/2eec6d4f10b34fcc8329e777e1cd9c78 2024-11-07T12:41:36,495 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/924be4e1ecfa4f6f9ae712ed2173478f to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/A/924be4e1ecfa4f6f9ae712ed2173478f 2024-11-07T12:41:36,500 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/3f2f0f23010a4ab9806523d5948ad83c to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/3f2f0f23010a4ab9806523d5948ad83c 2024-11-07T12:41:36,502 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/d63f47ef70b24757afd08ffef22bb8bd to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/B/d63f47ef70b24757afd08ffef22bb8bd 2024-11-07T12:41:36,506 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/2e08dab1cba64346a5ae1af0026de971 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/2e08dab1cba64346a5ae1af0026de971 2024-11-07T12:41:36,507 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/788a9361f1a84767996e29b71967c428 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/C/788a9361f1a84767996e29b71967c428 2024-11-07T12:41:36,511 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/recovered.edits/533.seqid to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe/recovered.edits/533.seqid 2024-11-07T12:41:36,511 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/13f50260f7a437ef753d61fda36692fe 2024-11-07T12:41:36,512 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-07T12:41:36,521 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=23, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:41:36,527 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-07T12:41:36,535 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-07T12:41:36,582 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-07T12:41:36,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-07T12:41:36,584 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=23, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:41:36,584 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-07T12:41:36,584 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1730983296584"}]},"ts":"9223372036854775807"} 2024-11-07T12:41:36,590 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-07T12:41:36,590 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 13f50260f7a437ef753d61fda36692fe, NAME => 'TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe.', STARTKEY => '', ENDKEY => ''}] 2024-11-07T12:41:36,590 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-07T12:41:36,590 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1730983296590"}]},"ts":"9223372036854775807"} 2024-11-07T12:41:36,594 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-07T12:41:36,595 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=23, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:41:36,597 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 128 msec 2024-11-07T12:41:36,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-07T12:41:36,792 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-07T12:41:36,792 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:41:36,806 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=245 (was 209) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-269572622_22 at /127.0.0.1:54914 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;db9ad1cb6cf9:34293-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1552 (was 1921), ProcessCount=11 (was 11), AvailableMemoryMB=3058 (was 4399) 2024-11-07T12:41:36,816 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=245, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=1552, ProcessCount=11, AvailableMemoryMB=3057 2024-11-07T12:41:36,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-07T12:41:36,818 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T12:41:36,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] procedure2.ProcedureExecutor(1139): Stored pid=24, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-07T12:41:36,821 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T12:41:36,821 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:36,821 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 24 2024-11-07T12:41:36,822 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T12:41:36,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=24 2024-11-07T12:41:36,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741936_1112 (size=1061) 2024-11-07T12:41:36,831 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4 2024-11-07T12:41:36,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741937_1113 (size=53) 2024-11-07T12:41:36,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=24 2024-11-07T12:41:37,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=24 2024-11-07T12:41:37,241 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:41:37,241 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 90c0d1c8af782719d9f8d037e72c2a12, disabling compactions & flushes 2024-11-07T12:41:37,241 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:37,241 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:37,241 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. after waiting 0 ms 2024-11-07T12:41:37,241 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:37,241 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:37,241 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 90c0d1c8af782719d9f8d037e72c2a12: Waiting for close lock at 1730983297241Disabling compacts and flushes for region at 1730983297241Disabling writes for close at 1730983297241Writing region close event to WAL at 1730983297241Closed at 1730983297241 2024-11-07T12:41:37,243 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T12:41:37,243 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1730983297243"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730983297243"}]},"ts":"1730983297243"} 2024-11-07T12:41:37,245 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-07T12:41:37,246 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T12:41:37,246 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730983297246"}]},"ts":"1730983297246"} 2024-11-07T12:41:37,248 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-07T12:41:37,248 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=90c0d1c8af782719d9f8d037e72c2a12, ASSIGN}] 2024-11-07T12:41:37,249 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=90c0d1c8af782719d9f8d037e72c2a12, ASSIGN 2024-11-07T12:41:37,252 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=90c0d1c8af782719d9f8d037e72c2a12, ASSIGN; state=OFFLINE, location=db9ad1cb6cf9,34293,1730983260217; forceNewPlan=false, retain=false 2024-11-07T12:41:37,403 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=90c0d1c8af782719d9f8d037e72c2a12, regionState=OPENING, regionLocation=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:37,405 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=90c0d1c8af782719d9f8d037e72c2a12, ASSIGN because future has completed 2024-11-07T12:41:37,406 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217}] 2024-11-07T12:41:37,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=24 2024-11-07T12:41:37,561 INFO [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:37,561 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(7752): Opening region: {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} 2024-11-07T12:41:37,562 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:41:37,562 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:41:37,562 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(7794): checking encryption for 90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:41:37,562 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(7797): checking classloading for 90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:41:37,563 INFO [StoreOpener-90c0d1c8af782719d9f8d037e72c2a12-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:41:37,565 INFO [StoreOpener-90c0d1c8af782719d9f8d037e72c2a12-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-07T12:41:37,565 INFO [StoreOpener-90c0d1c8af782719d9f8d037e72c2a12-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 90c0d1c8af782719d9f8d037e72c2a12 columnFamilyName A 2024-11-07T12:41:37,566 DEBUG [StoreOpener-90c0d1c8af782719d9f8d037e72c2a12-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:37,566 INFO [StoreOpener-90c0d1c8af782719d9f8d037e72c2a12-1 {}] regionserver.HStore(327): Store=90c0d1c8af782719d9f8d037e72c2a12/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:41:37,566 INFO [StoreOpener-90c0d1c8af782719d9f8d037e72c2a12-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:41:37,567 INFO [StoreOpener-90c0d1c8af782719d9f8d037e72c2a12-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-07T12:41:37,568 INFO [StoreOpener-90c0d1c8af782719d9f8d037e72c2a12-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 90c0d1c8af782719d9f8d037e72c2a12 columnFamilyName B 2024-11-07T12:41:37,568 DEBUG [StoreOpener-90c0d1c8af782719d9f8d037e72c2a12-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:37,568 INFO [StoreOpener-90c0d1c8af782719d9f8d037e72c2a12-1 {}] regionserver.HStore(327): Store=90c0d1c8af782719d9f8d037e72c2a12/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:41:37,568 INFO [StoreOpener-90c0d1c8af782719d9f8d037e72c2a12-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:41:37,569 INFO [StoreOpener-90c0d1c8af782719d9f8d037e72c2a12-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-07T12:41:37,570 INFO [StoreOpener-90c0d1c8af782719d9f8d037e72c2a12-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 90c0d1c8af782719d9f8d037e72c2a12 columnFamilyName C 2024-11-07T12:41:37,570 DEBUG [StoreOpener-90c0d1c8af782719d9f8d037e72c2a12-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:37,570 INFO [StoreOpener-90c0d1c8af782719d9f8d037e72c2a12-1 {}] regionserver.HStore(327): Store=90c0d1c8af782719d9f8d037e72c2a12/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:41:37,570 INFO [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:37,571 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1038): replaying wal for 90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:41:37,571 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:41:37,572 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:41:37,574 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1048): stopping wal replay for 90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:41:37,574 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1060): Cleaning up temporary data for 90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:41:37,574 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-07T12:41:37,576 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1093): writing seq id for 90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:41:37,578 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T12:41:37,579 INFO [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1114): Opened 90c0d1c8af782719d9f8d037e72c2a12; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60958463, jitterRate=-0.09164811670780182}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-07T12:41:37,579 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:41:37,580 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1006): Region open journal for 90c0d1c8af782719d9f8d037e72c2a12: Running coprocessor pre-open hook at 1730983297562Writing region info on filesystem at 1730983297562Initializing all the Stores at 1730983297563 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730983297563Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730983297563Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730983297563Cleaning up temporary data from old regions at 1730983297574 (+11 ms)Running coprocessor post-open hooks at 1730983297579 (+5 ms)Region opened successfully at 1730983297580 (+1 ms) 2024-11-07T12:41:37,581 INFO [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., pid=26, masterSystemTime=1730983297558 2024-11-07T12:41:37,583 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:37,583 INFO [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:37,584 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=90c0d1c8af782719d9f8d037e72c2a12, regionState=OPEN, openSeqNum=2, regionLocation=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:37,586 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 because future has completed 2024-11-07T12:41:37,589 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-07T12:41:37,589 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; OpenRegionProcedure 90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 in 181 msec 2024-11-07T12:41:37,592 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=25, resume processing ppid=24 2024-11-07T12:41:37,592 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=24, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=90c0d1c8af782719d9f8d037e72c2a12, ASSIGN in 341 msec 2024-11-07T12:41:37,593 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T12:41:37,593 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730983297593"}]},"ts":"1730983297593"} 2024-11-07T12:41:37,595 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-07T12:41:37,596 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T12:41:37,598 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 778 msec 2024-11-07T12:41:37,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=24 2024-11-07T12:41:37,963 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:41:37,964 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79527845, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:41:37,964 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request db9ad1cb6cf9,38619,-1 for getting cluster id 2024-11-07T12:41:37,964 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:41:37,965 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e542096-4b0e-4b01-9952-662501fc0f65' 2024-11-07T12:41:37,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:41:37,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e542096-4b0e-4b01-9952-662501fc0f65" 2024-11-07T12:41:37,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fe9465e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:41:37,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [db9ad1cb6cf9,38619,-1] 2024-11-07T12:41:37,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:41:37,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:41:37,968 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37028, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:41:37,968 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46d51a69, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:41:37,969 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:41:37,970 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=-1] 2024-11-07T12:41:37,970 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:41:37,971 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42980, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:41:37,972 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-07T12:41:37,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is db9ad1cb6cf9,38619,1730983259119 2024-11-07T12:41:37,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2e84b354 2024-11-07T12:41:37,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-07T12:41:37,974 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37030, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-07T12:41:37,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-07T12:41:37,980 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T12:41:37,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-07T12:41:38,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741938_1114 (size=1097) 2024-11-07T12:41:38,402 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-07T12:41:38,402 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-07T12:41:38,407 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-07T12:41:38,415 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=90c0d1c8af782719d9f8d037e72c2a12, REOPEN/MOVE}] 2024-11-07T12:41:38,416 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=90c0d1c8af782719d9f8d037e72c2a12, REOPEN/MOVE 2024-11-07T12:41:38,417 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=90c0d1c8af782719d9f8d037e72c2a12, regionState=CLOSING, regionLocation=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:38,419 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=90c0d1c8af782719d9f8d037e72c2a12, REOPEN/MOVE because future has completed 2024-11-07T12:41:38,420 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-07T12:41:38,420 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure 90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217}] 2024-11-07T12:41:38,573 INFO [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(122): Close 90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:41:38,574 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-07T12:41:38,574 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1722): Closing 90c0d1c8af782719d9f8d037e72c2a12, disabling compactions & flushes 2024-11-07T12:41:38,574 INFO [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:38,574 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:38,574 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. after waiting 0 ms 2024-11-07T12:41:38,574 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:38,578 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-07T12:41:38,579 INFO [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:38,579 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1676): Region close journal for 90c0d1c8af782719d9f8d037e72c2a12: Waiting for close lock at 1730983298574Running coprocessor pre-close hooks at 1730983298574Disabling compacts and flushes for region at 1730983298574Disabling writes for close at 1730983298574Writing region close event to WAL at 1730983298574Running coprocessor post-close hooks at 1730983298579 (+5 ms)Closed at 1730983298579 2024-11-07T12:41:38,580 WARN [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionServer(3299): Not adding moved region record: 90c0d1c8af782719d9f8d037e72c2a12 to self. 2024-11-07T12:41:38,582 INFO [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(157): Closed 90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:41:38,583 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=90c0d1c8af782719d9f8d037e72c2a12, regionState=CLOSED 2024-11-07T12:41:38,584 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure 90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 because future has completed 2024-11-07T12:41:38,587 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-07T12:41:38,587 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseRegionProcedure 90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 in 165 msec 2024-11-07T12:41:38,588 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=90c0d1c8af782719d9f8d037e72c2a12, REOPEN/MOVE; state=CLOSED, location=db9ad1cb6cf9,34293,1730983260217; forceNewPlan=false, retain=true 2024-11-07T12:41:38,739 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=90c0d1c8af782719d9f8d037e72c2a12, regionState=OPENING, regionLocation=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:38,741 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=90c0d1c8af782719d9f8d037e72c2a12, REOPEN/MOVE because future has completed 2024-11-07T12:41:38,742 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217}] 2024-11-07T12:41:38,897 INFO [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:38,898 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} 2024-11-07T12:41:38,898 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:41:38,898 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:41:38,898 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for 90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:41:38,898 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for 90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:41:38,904 INFO [StoreOpener-90c0d1c8af782719d9f8d037e72c2a12-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:41:38,904 INFO [StoreOpener-90c0d1c8af782719d9f8d037e72c2a12-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-07T12:41:38,911 INFO [StoreOpener-90c0d1c8af782719d9f8d037e72c2a12-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 90c0d1c8af782719d9f8d037e72c2a12 columnFamilyName A 2024-11-07T12:41:38,913 DEBUG [StoreOpener-90c0d1c8af782719d9f8d037e72c2a12-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:38,913 INFO [StoreOpener-90c0d1c8af782719d9f8d037e72c2a12-1 {}] regionserver.HStore(327): Store=90c0d1c8af782719d9f8d037e72c2a12/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:41:38,914 INFO [StoreOpener-90c0d1c8af782719d9f8d037e72c2a12-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:41:38,915 INFO [StoreOpener-90c0d1c8af782719d9f8d037e72c2a12-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-07T12:41:38,915 INFO [StoreOpener-90c0d1c8af782719d9f8d037e72c2a12-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 90c0d1c8af782719d9f8d037e72c2a12 columnFamilyName B 2024-11-07T12:41:38,915 DEBUG [StoreOpener-90c0d1c8af782719d9f8d037e72c2a12-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:38,917 INFO [StoreOpener-90c0d1c8af782719d9f8d037e72c2a12-1 {}] regionserver.HStore(327): Store=90c0d1c8af782719d9f8d037e72c2a12/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:41:38,917 INFO [StoreOpener-90c0d1c8af782719d9f8d037e72c2a12-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:41:38,917 INFO [StoreOpener-90c0d1c8af782719d9f8d037e72c2a12-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-07T12:41:38,918 INFO [StoreOpener-90c0d1c8af782719d9f8d037e72c2a12-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 90c0d1c8af782719d9f8d037e72c2a12 columnFamilyName C 2024-11-07T12:41:38,918 DEBUG [StoreOpener-90c0d1c8af782719d9f8d037e72c2a12-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:38,918 INFO [StoreOpener-90c0d1c8af782719d9f8d037e72c2a12-1 {}] regionserver.HStore(327): Store=90c0d1c8af782719d9f8d037e72c2a12/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:41:38,918 INFO [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:38,918 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for 90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:41:38,919 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:41:38,921 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:41:38,922 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for 90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:41:38,922 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for 90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:41:38,922 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-07T12:41:38,924 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for 90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:41:38,925 INFO [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened 90c0d1c8af782719d9f8d037e72c2a12; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66238588, jitterRate=-0.012968122959136963}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-07T12:41:38,925 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:41:38,928 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for 90c0d1c8af782719d9f8d037e72c2a12: Running coprocessor pre-open hook at 1730983298898Writing region info on filesystem at 1730983298898Initializing all the Stores at 1730983298899 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730983298899Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730983298899Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730983298900 (+1 ms)Cleaning up temporary data from old regions at 1730983298922 (+22 ms)Running coprocessor post-open hooks at 1730983298925 (+3 ms)Region opened successfully at 1730983298928 (+3 ms) 2024-11-07T12:41:38,929 INFO [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., pid=31, masterSystemTime=1730983298894 2024-11-07T12:41:38,932 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:38,932 INFO [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:38,932 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=90c0d1c8af782719d9f8d037e72c2a12, regionState=OPEN, openSeqNum=5, regionLocation=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:38,937 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 because future has completed 2024-11-07T12:41:38,949 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=29 2024-11-07T12:41:38,949 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 in 197 msec 2024-11-07T12:41:38,952 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-07T12:41:38,952 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=90c0d1c8af782719d9f8d037e72c2a12, REOPEN/MOVE in 534 msec 2024-11-07T12:41:38,956 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-07T12:41:38,956 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 546 msec 2024-11-07T12:41:38,960 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 976 msec 2024-11-07T12:41:38,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-07T12:41:38,964 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:41:38,965 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5adc4e02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:41:38,965 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request db9ad1cb6cf9,38619,-1 for getting cluster id 2024-11-07T12:41:38,965 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:41:38,966 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e542096-4b0e-4b01-9952-662501fc0f65' 2024-11-07T12:41:38,967 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:41:38,967 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e542096-4b0e-4b01-9952-662501fc0f65" 2024-11-07T12:41:38,967 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fc296a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:41:38,967 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [db9ad1cb6cf9,38619,-1] 2024-11-07T12:41:38,967 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:41:38,967 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:41:38,971 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37038, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:41:38,973 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3994f5a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:41:38,977 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@681c9b14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:41:38,977 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request db9ad1cb6cf9,38619,-1 for getting cluster id 2024-11-07T12:41:38,977 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:41:38,982 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e542096-4b0e-4b01-9952-662501fc0f65' 2024-11-07T12:41:38,988 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:41:38,989 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e542096-4b0e-4b01-9952-662501fc0f65" 2024-11-07T12:41:38,989 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53ed93f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:41:38,989 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [db9ad1cb6cf9,38619,-1] 2024-11-07T12:41:38,989 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:41:38,989 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:41:38,997 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37062, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:41:38,998 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cb46c59, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:41:39,001 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51d91e8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:41:39,001 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request db9ad1cb6cf9,38619,-1 for getting cluster id 2024-11-07T12:41:39,001 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:41:39,005 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e542096-4b0e-4b01-9952-662501fc0f65' 2024-11-07T12:41:39,005 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:41:39,005 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e542096-4b0e-4b01-9952-662501fc0f65" 2024-11-07T12:41:39,005 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66747aff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:41:39,005 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [db9ad1cb6cf9,38619,-1] 2024-11-07T12:41:39,006 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:41:39,006 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:41:39,009 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37082, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:41:39,010 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d119f6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:41:39,014 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@771f5e02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:41:39,014 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request db9ad1cb6cf9,38619,-1 for getting cluster id 2024-11-07T12:41:39,014 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:41:39,016 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e542096-4b0e-4b01-9952-662501fc0f65' 2024-11-07T12:41:39,016 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:41:39,016 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e542096-4b0e-4b01-9952-662501fc0f65" 2024-11-07T12:41:39,016 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15fe1248, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:41:39,016 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [db9ad1cb6cf9,38619,-1] 2024-11-07T12:41:39,025 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:41:39,026 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:41:39,029 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37106, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:41:39,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71f0a76f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:41:39,037 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37b75e2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:41:39,037 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request db9ad1cb6cf9,38619,-1 for getting cluster id 2024-11-07T12:41:39,037 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:41:39,038 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e542096-4b0e-4b01-9952-662501fc0f65' 2024-11-07T12:41:39,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:41:39,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e542096-4b0e-4b01-9952-662501fc0f65" 2024-11-07T12:41:39,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@372130d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:41:39,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [db9ad1cb6cf9,38619,-1] 2024-11-07T12:41:39,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:41:39,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:41:39,047 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37134, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:41:39,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1970270a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:41:39,053 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f5453a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:41:39,053 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request db9ad1cb6cf9,38619,-1 for getting cluster id 2024-11-07T12:41:39,053 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:41:39,054 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e542096-4b0e-4b01-9952-662501fc0f65' 2024-11-07T12:41:39,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:41:39,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e542096-4b0e-4b01-9952-662501fc0f65" 2024-11-07T12:41:39,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@242a10d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:41:39,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [db9ad1cb6cf9,38619,-1] 2024-11-07T12:41:39,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:41:39,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:41:39,057 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37140, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:41:39,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11a60302, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:41:39,073 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73b6e690, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:41:39,073 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request db9ad1cb6cf9,38619,-1 for getting cluster id 2024-11-07T12:41:39,073 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:41:39,074 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e542096-4b0e-4b01-9952-662501fc0f65' 2024-11-07T12:41:39,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:41:39,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e542096-4b0e-4b01-9952-662501fc0f65" 2024-11-07T12:41:39,075 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12766e2d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:41:39,075 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [db9ad1cb6cf9,38619,-1] 2024-11-07T12:41:39,075 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:41:39,075 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:41:39,076 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37158, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:41:39,077 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ffefa90, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:41:39,080 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bd05ea8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:41:39,080 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request db9ad1cb6cf9,38619,-1 for getting cluster id 2024-11-07T12:41:39,081 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:41:39,081 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e542096-4b0e-4b01-9952-662501fc0f65' 2024-11-07T12:41:39,082 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:41:39,082 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e542096-4b0e-4b01-9952-662501fc0f65" 2024-11-07T12:41:39,082 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e67e0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:41:39,082 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [db9ad1cb6cf9,38619,-1] 2024-11-07T12:41:39,082 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:41:39,082 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:41:39,084 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37180, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:41:39,084 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@555ec6ee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:41:39,085 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dff183a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:41:39,085 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request db9ad1cb6cf9,38619,-1 for getting cluster id 2024-11-07T12:41:39,086 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:41:39,087 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e542096-4b0e-4b01-9952-662501fc0f65' 2024-11-07T12:41:39,087 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:41:39,087 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e542096-4b0e-4b01-9952-662501fc0f65" 2024-11-07T12:41:39,087 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46c00aec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:41:39,087 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [db9ad1cb6cf9,38619,-1] 2024-11-07T12:41:39,087 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:41:39,088 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:41:39,089 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37202, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:41:39,090 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44ef1482, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:41:39,093 DEBUG [Thread-593 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:41:39,093 DEBUG [Thread-595 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:41:39,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=-1] 2024-11-07T12:41:39,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=-1] 2024-11-07T12:41:39,095 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:41:39,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:41:39,096 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43000, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:41:39,097 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42994, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:41:39,100 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5] 2024-11-07T12:41:39,101 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5] 2024-11-07T12:41:39,105 DEBUG [Thread-604 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:41:39,106 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=-1] 2024-11-07T12:41:39,107 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:41:39,108 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43006, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:41:39,110 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5] 2024-11-07T12:41:39,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:41:39,127 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 90c0d1c8af782719d9f8d037e72c2a12 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-07T12:41:39,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 90c0d1c8af782719d9f8d037e72c2a12, store=A 2024-11-07T12:41:39,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:39,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 90c0d1c8af782719d9f8d037e72c2a12, store=B 2024-11-07T12:41:39,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:39,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 90c0d1c8af782719d9f8d037e72c2a12, store=C 2024-11-07T12:41:39,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:39,129 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:41:39,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] procedure2.ProcedureExecutor(1139): Stored pid=32, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=32, table=TestAcidGuarantees 2024-11-07T12:41:39,132 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=32, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=32, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:41:39,133 DEBUG [Thread-600 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:41:39,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-07T12:41:39,134 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=32, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=32, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:41:39,134 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:41:39,135 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=-1] 2024-11-07T12:41:39,135 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:41:39,137 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43016, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:41:39,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5] 2024-11-07T12:41:39,141 DEBUG [Thread-587 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:41:39,145 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=-1] 2024-11-07T12:41:39,145 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:41:39,149 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43028, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:41:39,150 DEBUG [Thread-598 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:41:39,151 DEBUG [Thread-598 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=-1] 2024-11-07T12:41:39,151 DEBUG [Thread-598 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:41:39,152 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5] 2024-11-07T12:41:39,153 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43034, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:41:39,155 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5] 2024-11-07T12:41:39,163 DEBUG [Thread-589 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:41:39,166 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=-1] 2024-11-07T12:41:39,166 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:41:39,167 DEBUG [Thread-602 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:41:39,168 DEBUG [Thread-591 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:41:39,168 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43038, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:41:39,169 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=-1] 2024-11-07T12:41:39,169 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:41:39,171 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5] 2024-11-07T12:41:39,171 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43054, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:41:39,171 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=-1] 2024-11-07T12:41:39,171 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:41:39,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5] 2024-11-07T12:41:39,173 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43062, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:41:39,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5] 2024-11-07T12:41:39,202 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107778b5e8a3d974b19a5459b7379be1339_90c0d1c8af782719d9f8d037e72c2a12 is 50, key is test_row_0/A:col10/1730983299125/Put/seqid=0 2024-11-07T12:41:39,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741939_1115 (size=12154) 2024-11-07T12:41:39,233 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1730983299172, max timestamp=1730983299225Num uniques 859; 2024-11-07T12:41:39,233 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-07T12:41:39,233 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1730983299172, max timestamp=1730983299226Num uniques 859; 2024-11-07T12:41:39,233 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-07T12:41:39,233 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1730983299172, max timestamp=1730983299226Num uniques 859; 2024-11-07T12:41:39,234 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-07T12:41:39,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-07T12:41:39,289 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-07T12:41:39,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:39,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. as already flushing 2024-11-07T12:41:39,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:39,292 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:39,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:39,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:39,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:39,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42994 deadline: 1730983359283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:39,301 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:39,302 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:39,302 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:39,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:39,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43028 deadline: 1730983359287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:39,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:39,306 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:39,306 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:39,306 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:39,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43000 deadline: 1730983359289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:39,307 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:39,307 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:39,307 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:39,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:39,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43038 deadline: 1730983359298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:39,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:39,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43062 deadline: 1730983359301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:39,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:39,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:39,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:39,320 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:39,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:39,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:39,422 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:39,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42994 deadline: 1730983359413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:39,422 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:39,422 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:39,423 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:39,423 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:39,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43028 deadline: 1730983359413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:39,423 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:39,423 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:39,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:39,423 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:39,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43000 deadline: 1730983359414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:39,424 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:39,424 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:39,424 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:39,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:39,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43062 deadline: 1730983359422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:39,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:39,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:39,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:39,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:39,447 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-07T12:41:39,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43038 deadline: 1730983359434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:39,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:39,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. as already flushing 2024-11-07T12:41:39,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:39,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:39,448 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:39,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:39,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:39,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:39,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-07T12:41:39,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:39,612 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-07T12:41:39,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:39,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. as already flushing 2024-11-07T12:41:39,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:39,612 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:39,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:39,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:39,618 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:39,629 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107778b5e8a3d974b19a5459b7379be1339_90c0d1c8af782719d9f8d037e72c2a12 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107778b5e8a3d974b19a5459b7379be1339_90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:41:39,637 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/e57a80a0dfa84e38bc21cb6118c2e7b3, store: [table=TestAcidGuarantees family=A region=90c0d1c8af782719d9f8d037e72c2a12] 2024-11-07T12:41:39,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:39,648 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/e57a80a0dfa84e38bc21cb6118c2e7b3 is 175, key is test_row_0/A:col10/1730983299125/Put/seqid=0 2024-11-07T12:41:39,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42994 deadline: 1730983359633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:39,650 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:39,650 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:39,650 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:39,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:39,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43028 deadline: 1730983359633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:39,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:39,652 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:39,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43000 deadline: 1730983359633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:39,652 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:39,652 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:39,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:39,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:39,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:39,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:39,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43038 deadline: 1730983359652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:39,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:39,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:39,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:39,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:39,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43062 deadline: 1730983359654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:39,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:39,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:39,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:39,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741940_1116 (size=30955) 2024-11-07T12:41:39,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-07T12:41:39,766 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-07T12:41:39,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:39,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. as already flushing 2024-11-07T12:41:39,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:39,767 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:39,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:39,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:39,920 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-07T12:41:39,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:39,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. as already flushing 2024-11-07T12:41:39,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:39,920 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:39,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:39,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:39,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:39,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42994 deadline: 1730983359953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:39,965 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:39,965 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:39,965 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:39,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:39,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43028 deadline: 1730983359962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:39,969 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:39,969 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:39,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:39,969 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:39,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43000 deadline: 1730983359962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:39,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:39,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:39,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:39,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:39,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43062 deadline: 1730983359962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:39,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:39,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43038 deadline: 1730983359971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:39,980 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:39,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:39,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:39,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:39,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:39,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:40,076 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-07T12:41:40,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:40,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. as already flushing 2024-11-07T12:41:40,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:40,076 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:40,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:40,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:40,083 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/e57a80a0dfa84e38bc21cb6118c2e7b3 2024-11-07T12:41:40,131 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/ad2697a5acd64bac8322b00b13969389 is 50, key is test_row_0/B:col10/1730983299125/Put/seqid=0 2024-11-07T12:41:40,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741941_1117 (size=12001) 2024-11-07T12:41:40,230 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-07T12:41:40,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:40,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. as already flushing 2024-11-07T12:41:40,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:40,231 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:40,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:40,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:40,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-07T12:41:40,385 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-07T12:41:40,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:40,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. as already flushing 2024-11-07T12:41:40,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:40,389 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:40,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:40,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:40,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:40,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42994 deadline: 1730983360473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:40,478 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:40,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:40,478 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:40,478 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:40,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43028 deadline: 1730983360473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:40,479 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:40,479 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:40,479 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:40,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:40,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43000 deadline: 1730983360485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:40,489 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:40,489 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:40,489 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:40,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:40,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43038 deadline: 1730983360493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:40,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:40,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:40,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:40,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:40,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43062 deadline: 1730983360493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:40,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:40,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:40,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:40,543 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-07T12:41:40,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:40,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. as already flushing 2024-11-07T12:41:40,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:40,544 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:40,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:40,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:40,568 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/ad2697a5acd64bac8322b00b13969389 2024-11-07T12:41:40,627 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/47ce9bc6ffe44d3eb1ace53f8d626150 is 50, key is test_row_0/C:col10/1730983299125/Put/seqid=0 2024-11-07T12:41:40,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741942_1118 (size=12001) 2024-11-07T12:41:40,701 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-07T12:41:40,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:40,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. as already flushing 2024-11-07T12:41:40,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:40,702 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:40,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:40,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:40,865 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-07T12:41:40,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:40,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. as already flushing 2024-11-07T12:41:40,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:40,865 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:40,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:40,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:41,021 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-07T12:41:41,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:41,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. as already flushing 2024-11-07T12:41:41,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:41,022 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:41,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:41,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:41,085 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/47ce9bc6ffe44d3eb1ace53f8d626150 2024-11-07T12:41:41,130 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/e57a80a0dfa84e38bc21cb6118c2e7b3 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/e57a80a0dfa84e38bc21cb6118c2e7b3 2024-11-07T12:41:41,146 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/e57a80a0dfa84e38bc21cb6118c2e7b3, entries=150, sequenceid=14, filesize=30.2 K 2024-11-07T12:41:41,147 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/ad2697a5acd64bac8322b00b13969389 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/ad2697a5acd64bac8322b00b13969389 2024-11-07T12:41:41,166 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/ad2697a5acd64bac8322b00b13969389, entries=150, sequenceid=14, filesize=11.7 K 2024-11-07T12:41:41,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/47ce9bc6ffe44d3eb1ace53f8d626150 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/47ce9bc6ffe44d3eb1ace53f8d626150 2024-11-07T12:41:41,175 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-07T12:41:41,175 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/47ce9bc6ffe44d3eb1ace53f8d626150, entries=150, sequenceid=14, filesize=11.7 K 2024-11-07T12:41:41,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:41,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. as already flushing 2024-11-07T12:41:41,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:41,176 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:41,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:41,177 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 90c0d1c8af782719d9f8d037e72c2a12 in 2050ms, sequenceid=14, compaction requested=false 2024-11-07T12:41:41,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 90c0d1c8af782719d9f8d037e72c2a12: 2024-11-07T12:41:41,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:41,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-07T12:41:41,333 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-07T12:41:41,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:41,334 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2902): Flushing 90c0d1c8af782719d9f8d037e72c2a12 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-07T12:41:41,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 90c0d1c8af782719d9f8d037e72c2a12, store=A 2024-11-07T12:41:41,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:41:41,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 90c0d1c8af782719d9f8d037e72c2a12, store=B 2024-11-07T12:41:41,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:41:41,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 90c0d1c8af782719d9f8d037e72c2a12, store=C 2024-11-07T12:41:41,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:41:41,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110733404f76130445139df5da0c20d14d1e_90c0d1c8af782719d9f8d037e72c2a12 is 50, key is test_row_0/A:col10/1730983299248/Put/seqid=0 2024-11-07T12:41:41,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741943_1119 (size=12154) 2024-11-07T12:41:41,510 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T12:41:41,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:41:41,512 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. as already flushing 2024-11-07T12:41:41,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:41,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43000 deadline: 1730983361522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:41,538 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:41,538 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:41,538 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:41,545 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:41,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43028 deadline: 1730983361532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:41,545 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:41,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43062 deadline: 1730983361533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:41,546 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:41,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:41,546 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:41,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:41,546 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:41,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:41,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:41,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42994 deadline: 1730983361546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:41,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:41,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43038 deadline: 1730983361546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:41,552 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:41,552 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:41,552 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:41,552 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:41,552 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:41,552 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:41,659 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:41,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43062 deadline: 1730983361652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:41,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:41,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43028 deadline: 1730983361655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:41,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:41,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:41,662 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:41,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:41,662 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:41,662 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:41,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:41,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43000 deadline: 1730983361649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:41,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:41,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43038 deadline: 1730983361662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:41,665 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:41,665 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:41,665 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:41,668 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:41,669 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:41,669 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:41,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:41,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42994 deadline: 1730983361663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:41,670 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:41,670 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:41,670 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:41,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:41,881 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110733404f76130445139df5da0c20d14d1e_90c0d1c8af782719d9f8d037e72c2a12 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110733404f76130445139df5da0c20d14d1e_90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:41:41,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:41,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43038 deadline: 1730983361875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:41,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/91f8ed42a5fc4d7db673a90a75de4aa6, store: [table=TestAcidGuarantees family=A region=90c0d1c8af782719d9f8d037e72c2a12] 2024-11-07T12:41:41,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:41,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43062 deadline: 1730983361876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:41,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/91f8ed42a5fc4d7db673a90a75de4aa6 is 175, key is test_row_0/A:col10/1730983299248/Put/seqid=0 2024-11-07T12:41:41,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:41,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:41,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:41,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:41,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43028 deadline: 1730983361877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:41,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:41,887 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:41,887 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:41,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43000 deadline: 1730983361877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:41,887 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:41,888 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:41,888 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:41,888 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:41,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:41,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42994 deadline: 1730983361885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:41,888 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:41,888 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:41,888 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:41,888 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:41,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:41,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:41,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741944_1120 (size=30955) 2024-11-07T12:41:42,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:42,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43028 deadline: 1730983362197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:42,198 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:42,198 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:42,198 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:42,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:42,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42994 deadline: 1730983362197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:42,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:42,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43000 deadline: 1730983362197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:42,200 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:42,200 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:42,201 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:42,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:42,201 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:42,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43062 deadline: 1730983362197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:42,201 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:42,201 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:42,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:42,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:42,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:42,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:42,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43038 deadline: 1730983362208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:42,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:42,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:42,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:42,354 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/91f8ed42a5fc4d7db673a90a75de4aa6 2024-11-07T12:41:42,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/f52a2415307c48cc980c0c3dcfc4eda3 is 50, key is test_row_0/B:col10/1730983299248/Put/seqid=0 2024-11-07T12:41:42,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741945_1121 (size=12001) 2024-11-07T12:41:42,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:42,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43062 deadline: 1730983362714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:42,724 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:42,724 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:42,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:42,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:42,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42994 deadline: 1730983362716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:42,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:42,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43028 deadline: 1730983362716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:42,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:42,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43000 deadline: 1730983362717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:42,730 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:42,730 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:42,731 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:42,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:42,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43038 deadline: 1730983362726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:42,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:42,733 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:42,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:42,733 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:42,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:42,733 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:42,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:42,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:42,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:42,855 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/f52a2415307c48cc980c0c3dcfc4eda3 2024-11-07T12:41:42,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/c50b605f280044f59c4ae0341ced5511 is 50, key is test_row_0/C:col10/1730983299248/Put/seqid=0 2024-11-07T12:41:42,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741946_1122 (size=12001) 2024-11-07T12:41:42,929 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/c50b605f280044f59c4ae0341ced5511 2024-11-07T12:41:42,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/91f8ed42a5fc4d7db673a90a75de4aa6 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/91f8ed42a5fc4d7db673a90a75de4aa6 2024-11-07T12:41:42,955 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/91f8ed42a5fc4d7db673a90a75de4aa6, entries=150, sequenceid=51, filesize=30.2 K 2024-11-07T12:41:42,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/f52a2415307c48cc980c0c3dcfc4eda3 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/f52a2415307c48cc980c0c3dcfc4eda3 2024-11-07T12:41:42,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:42,966 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/f52a2415307c48cc980c0c3dcfc4eda3, entries=150, sequenceid=51, filesize=11.7 K 2024-11-07T12:41:42,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/c50b605f280044f59c4ae0341ced5511 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/c50b605f280044f59c4ae0341ced5511 2024-11-07T12:41:42,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:42,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:42,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:42,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:42,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:42,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:42,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:42,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:42,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:42,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:42,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:42,986 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/c50b605f280044f59c4ae0341ced5511, entries=150, sequenceid=51, filesize=11.7 K 2024-11-07T12:41:42,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:42,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:42,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:42,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:42,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:42,993 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 90c0d1c8af782719d9f8d037e72c2a12 in 1658ms, sequenceid=51, compaction requested=false 2024-11-07T12:41:42,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:42,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2603): Flush status journal for 90c0d1c8af782719d9f8d037e72c2a12: 2024-11-07T12:41:42,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:42,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=33 2024-11-07T12:41:42,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:42,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:42,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:42,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4169): Remote procedure done, pid=33 2024-11-07T12:41:42,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:42,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:42,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:42,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:42,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:42,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:42,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:42,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:42,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:42,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:42,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,008 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-07T12:41:43,008 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.8620 sec 2024-11-07T12:41:43,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,014 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=32, table=TestAcidGuarantees in 3.8800 sec 2024-11-07T12:41:43,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-07T12:41:43,302 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:41:43,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,302 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:41:43,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] procedure2.ProcedureExecutor(1139): Stored pid=34, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees 2024-11-07T12:41:43,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,305 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=34, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:41:43,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-07T12:41:43,306 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=34, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:41:43,306 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:41:43,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-07T12:41:43,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,461 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-07T12:41:43,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:43,462 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2902): Flushing 90c0d1c8af782719d9f8d037e72c2a12 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-07T12:41:43,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 90c0d1c8af782719d9f8d037e72c2a12, store=A 2024-11-07T12:41:43,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:43,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 90c0d1c8af782719d9f8d037e72c2a12, store=B 2024-11-07T12:41:43,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:43,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 90c0d1c8af782719d9f8d037e72c2a12, store=C 2024-11-07T12:41:43,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:43,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107e607456ef5894f76b4d6c660fc411a02_90c0d1c8af782719d9f8d037e72c2a12 is 50, key is test_row_0/A:col10/1730983301533/Put/seqid=0 2024-11-07T12:41:43,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741947_1123 (size=9714) 2024-11-07T12:41:43,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,528 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107e607456ef5894f76b4d6c660fc411a02_90c0d1c8af782719d9f8d037e72c2a12 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107e607456ef5894f76b4d6c660fc411a02_90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:41:43,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/101b717dc16c43ddbd91cf876eae76c3, store: [table=TestAcidGuarantees family=A region=90c0d1c8af782719d9f8d037e72c2a12] 2024-11-07T12:41:43,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/101b717dc16c43ddbd91cf876eae76c3 is 175, key is test_row_0/A:col10/1730983301533/Put/seqid=0 2024-11-07T12:41:43,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741948_1124 (size=22361) 2024-11-07T12:41:43,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,557 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=61, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/101b717dc16c43ddbd91cf876eae76c3 2024-11-07T12:41:43,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/f1437c3c37174f49b4aa4fe48b3098b3 is 50, key is test_row_0/B:col10/1730983301533/Put/seqid=0 2024-11-07T12:41:43,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741949_1125 (size=9657) 2024-11-07T12:41:43,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,610 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/f1437c3c37174f49b4aa4fe48b3098b3 2024-11-07T12:41:43,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-07T12:41:43,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/191f690638bf42758c47af0fbc73bc03 is 50, key is test_row_0/C:col10/1730983301533/Put/seqid=0 2024-11-07T12:41:43,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741950_1126 (size=9657) 2024-11-07T12:41:43,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,691 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/191f690638bf42758c47af0fbc73bc03 2024-11-07T12:41:43,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/101b717dc16c43ddbd91cf876eae76c3 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/101b717dc16c43ddbd91cf876eae76c3 2024-11-07T12:41:43,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,717 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/101b717dc16c43ddbd91cf876eae76c3, entries=100, sequenceid=61, filesize=21.8 K 2024-11-07T12:41:43,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/f1437c3c37174f49b4aa4fe48b3098b3 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/f1437c3c37174f49b4aa4fe48b3098b3 2024-11-07T12:41:43,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,725 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/f1437c3c37174f49b4aa4fe48b3098b3, entries=100, sequenceid=61, filesize=9.4 K 2024-11-07T12:41:43,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/191f690638bf42758c47af0fbc73bc03 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/191f690638bf42758c47af0fbc73bc03 2024-11-07T12:41:43,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,735 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/191f690638bf42758c47af0fbc73bc03, entries=100, sequenceid=61, filesize=9.4 K 2024-11-07T12:41:43,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,739 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=0 B/0 for 90c0d1c8af782719d9f8d037e72c2a12 in 278ms, sequenceid=61, compaction requested=true 2024-11-07T12:41:43,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2603): Flush status journal for 90c0d1c8af782719d9f8d037e72c2a12: 2024-11-07T12:41:43,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:43,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=35 2024-11-07T12:41:43,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4169): Remote procedure done, pid=35 2024-11-07T12:41:43,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,767 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-07T12:41:43,767 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 442 msec 2024-11-07T12:41:43,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,771 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees in 465 msec 2024-11-07T12:41:43,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,854 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 90c0d1c8af782719d9f8d037e72c2a12 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-07T12:41:43,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 90c0d1c8af782719d9f8d037e72c2a12, store=A 2024-11-07T12:41:43,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:43,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 90c0d1c8af782719d9f8d037e72c2a12, store=B 2024-11-07T12:41:43,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:43,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 90c0d1c8af782719d9f8d037e72c2a12, store=C 2024-11-07T12:41:43,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:43,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:41:43,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,895 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107755bb953d4f748138c3beef211638ef7_90c0d1c8af782719d9f8d037e72c2a12 is 50, key is test_row_0/A:col10/1730983303833/Put/seqid=0 2024-11-07T12:41:43,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:43,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741951_1127 (size=19474) 2024-11-07T12:41:43,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-07T12:41:43,932 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:41:43,932 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:41:43,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees 2024-11-07T12:41:43,935 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:41:43,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-07T12:41:43,936 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:41:43,936 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:41:43,947 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1730983303916, max timestamp=1730983303942Num uniques 859; 2024-11-07T12:41:43,947 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-07T12:41:43,947 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1730983303916, max timestamp=1730983303942Num uniques 859; 2024-11-07T12:41:43,947 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-07T12:41:43,950 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1730983303916, max timestamp=1730983303942Num uniques 859; 2024-11-07T12:41:43,950 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-07T12:41:44,000 INFO [master/db9ad1cb6cf9:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-07T12:41:44,000 INFO [master/db9ad1cb6cf9:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-07T12:41:44,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:44,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43000 deadline: 1730983363994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:44,004 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:44,004 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:44,004 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:44,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:44,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43028 deadline: 1730983363995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:44,010 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:44,010 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:44,010 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:44,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42994 deadline: 1730983363996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:44,010 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:44,010 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:44,010 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:44,010 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:44,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:44,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43062 deadline: 1730983364002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:44,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:44,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:44,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:44,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:44,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43038 deadline: 1730983364003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:44,012 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:44,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:44,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:44,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-07T12:41:44,090 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-07T12:41:44,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:44,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. as already flushing 2024-11-07T12:41:44,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:44,090 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:44,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:44,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:44,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:44,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43000 deadline: 1730983364117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:44,119 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:44,119 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:44,119 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:44,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:44,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43028 deadline: 1730983364122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:44,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:44,123 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:44,123 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:44,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42994 deadline: 1730983364122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:44,123 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:44,124 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:44,124 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:44,124 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:44,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:44,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43038 deadline: 1730983364122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:44,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:44,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:44,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:44,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43062 deadline: 1730983364122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:44,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:44,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:44,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:44,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:44,244 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-07T12:41:44,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:44,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. as already flushing 2024-11-07T12:41:44,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:44,246 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:44,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:44,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:44,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-07T12:41:44,323 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:44,328 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107755bb953d4f748138c3beef211638ef7_90c0d1c8af782719d9f8d037e72c2a12 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107755bb953d4f748138c3beef211638ef7_90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:41:44,331 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/fa30f245fa58420bacea55f4c1e8ba56, store: [table=TestAcidGuarantees family=A region=90c0d1c8af782719d9f8d037e72c2a12] 2024-11-07T12:41:44,331 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/fa30f245fa58420bacea55f4c1e8ba56 is 175, key is test_row_0/A:col10/1730983303833/Put/seqid=0 2024-11-07T12:41:44,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:44,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43000 deadline: 1730983364321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:44,337 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:44,337 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:44,337 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:44,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:44,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43028 deadline: 1730983364332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:44,339 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:44,339 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:44,339 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:44,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:44,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42994 deadline: 1730983364332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:44,340 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:44,340 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:44,340 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:44,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:44,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43038 deadline: 1730983364342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:44,345 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:44,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:44,345 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:44,345 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:44,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43062 deadline: 1730983364342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:44,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:44,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:44,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:44,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741952_1128 (size=56733) 2024-11-07T12:41:44,374 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=71, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/fa30f245fa58420bacea55f4c1e8ba56 2024-11-07T12:41:44,397 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/4a3b2d7d850741958a8518dbf7184188 is 50, key is test_row_0/B:col10/1730983303833/Put/seqid=0 2024-11-07T12:41:44,404 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-07T12:41:44,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:44,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. as already flushing 2024-11-07T12:41:44,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:44,405 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:44,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:44,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:44,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741953_1129 (size=12001) 2024-11-07T12:41:44,434 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=71 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/4a3b2d7d850741958a8518dbf7184188 2024-11-07T12:41:44,454 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/8502002f71db4430a42f696609efdac6 is 50, key is test_row_0/C:col10/1730983303833/Put/seqid=0 2024-11-07T12:41:44,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741954_1130 (size=12001) 2024-11-07T12:41:44,502 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=71 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/8502002f71db4430a42f696609efdac6 2024-11-07T12:41:44,509 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/fa30f245fa58420bacea55f4c1e8ba56 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/fa30f245fa58420bacea55f4c1e8ba56 2024-11-07T12:41:44,515 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/fa30f245fa58420bacea55f4c1e8ba56, entries=300, sequenceid=71, filesize=55.4 K 2024-11-07T12:41:44,516 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/4a3b2d7d850741958a8518dbf7184188 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/4a3b2d7d850741958a8518dbf7184188 2024-11-07T12:41:44,525 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/4a3b2d7d850741958a8518dbf7184188, entries=150, sequenceid=71, filesize=11.7 K 2024-11-07T12:41:44,527 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/8502002f71db4430a42f696609efdac6 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/8502002f71db4430a42f696609efdac6 2024-11-07T12:41:44,546 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/8502002f71db4430a42f696609efdac6, entries=150, sequenceid=71, filesize=11.7 K 2024-11-07T12:41:44,548 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 90c0d1c8af782719d9f8d037e72c2a12 in 694ms, sequenceid=71, compaction requested=true 2024-11-07T12:41:44,549 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 90c0d1c8af782719d9f8d037e72c2a12: 2024-11-07T12:41:44,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 90c0d1c8af782719d9f8d037e72c2a12:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:41:44,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:41:44,549 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:41:44,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 90c0d1c8af782719d9f8d037e72c2a12:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:41:44,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:41:44,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 90c0d1c8af782719d9f8d037e72c2a12:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:41:44,549 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:41:44,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:41:44,551 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:41:44,551 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:41:44,551 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 90c0d1c8af782719d9f8d037e72c2a12/B is initiating minor compaction (all files) 2024-11-07T12:41:44,551 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): 90c0d1c8af782719d9f8d037e72c2a12/A is initiating minor compaction (all files) 2024-11-07T12:41:44,551 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 90c0d1c8af782719d9f8d037e72c2a12/B in TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:44,551 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 90c0d1c8af782719d9f8d037e72c2a12/A in TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:44,551 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/ad2697a5acd64bac8322b00b13969389, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/f52a2415307c48cc980c0c3dcfc4eda3, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/f1437c3c37174f49b4aa4fe48b3098b3, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/4a3b2d7d850741958a8518dbf7184188] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp, totalSize=44.6 K 2024-11-07T12:41:44,551 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/e57a80a0dfa84e38bc21cb6118c2e7b3, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/91f8ed42a5fc4d7db673a90a75de4aa6, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/101b717dc16c43ddbd91cf876eae76c3, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/fa30f245fa58420bacea55f4c1e8ba56] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp, totalSize=137.7 K 2024-11-07T12:41:44,551 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:44,552 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. files: [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/e57a80a0dfa84e38bc21cb6118c2e7b3, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/91f8ed42a5fc4d7db673a90a75de4aa6, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/101b717dc16c43ddbd91cf876eae76c3, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/fa30f245fa58420bacea55f4c1e8ba56] 2024-11-07T12:41:44,553 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting ad2697a5acd64bac8322b00b13969389, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1730983299102 2024-11-07T12:41:44,553 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting e57a80a0dfa84e38bc21cb6118c2e7b3, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1730983299102 2024-11-07T12:41:44,553 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting f52a2415307c48cc980c0c3dcfc4eda3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1730983299248 2024-11-07T12:41:44,553 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91f8ed42a5fc4d7db673a90a75de4aa6, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1730983299248 2024-11-07T12:41:44,554 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting f1437c3c37174f49b4aa4fe48b3098b3, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1730983301527 2024-11-07T12:41:44,555 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 101b717dc16c43ddbd91cf876eae76c3, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1730983301527 2024-11-07T12:41:44,555 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a3b2d7d850741958a8518dbf7184188, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=71, earliestPutTs=1730983303833 2024-11-07T12:41:44,555 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa30f245fa58420bacea55f4c1e8ba56, keycount=300, bloomtype=ROW, size=55.4 K, encoding=NONE, compression=NONE, seqNum=71, earliestPutTs=1730983303829 2024-11-07T12:41:44,561 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-07T12:41:44,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-07T12:41:44,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:44,565 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2902): Flushing 90c0d1c8af782719d9f8d037e72c2a12 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-07T12:41:44,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 90c0d1c8af782719d9f8d037e72c2a12, store=A 2024-11-07T12:41:44,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:41:44,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 90c0d1c8af782719d9f8d037e72c2a12, store=B 2024-11-07T12:41:44,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:41:44,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 90c0d1c8af782719d9f8d037e72c2a12, store=C 2024-11-07T12:41:44,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:41:44,577 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=90c0d1c8af782719d9f8d037e72c2a12] 2024-11-07T12:41:44,577 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 90c0d1c8af782719d9f8d037e72c2a12#B#compaction#111 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:41:44,578 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/ecdc018dd9194b0d8fe83793d52159a8 is 50, key is test_row_0/B:col10/1730983303833/Put/seqid=0 2024-11-07T12:41:44,601 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411071f13d7da8ed5480a8966e72ddeb14042_90c0d1c8af782719d9f8d037e72c2a12 store=[table=TestAcidGuarantees family=A region=90c0d1c8af782719d9f8d037e72c2a12] 2024-11-07T12:41:44,605 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411071f13d7da8ed5480a8966e72ddeb14042_90c0d1c8af782719d9f8d037e72c2a12, store=[table=TestAcidGuarantees family=A region=90c0d1c8af782719d9f8d037e72c2a12] 2024-11-07T12:41:44,605 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411071f13d7da8ed5480a8966e72ddeb14042_90c0d1c8af782719d9f8d037e72c2a12 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=90c0d1c8af782719d9f8d037e72c2a12] 2024-11-07T12:41:44,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107a6cb6a6f326d41a6a01e48905a25ae8c_90c0d1c8af782719d9f8d037e72c2a12 is 50, key is test_row_0/A:col10/1730983303950/Put/seqid=0 2024-11-07T12:41:44,647 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. as already flushing 2024-11-07T12:41:44,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:41:44,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741955_1131 (size=12139) 2024-11-07T12:41:44,661 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/ecdc018dd9194b0d8fe83793d52159a8 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/ecdc018dd9194b0d8fe83793d52159a8 2024-11-07T12:41:44,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:44,680 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 90c0d1c8af782719d9f8d037e72c2a12/B of 90c0d1c8af782719d9f8d037e72c2a12 into ecdc018dd9194b0d8fe83793d52159a8(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:41:44,681 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 90c0d1c8af782719d9f8d037e72c2a12: 2024-11-07T12:41:44,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43038 deadline: 1730983364670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:44,681 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., storeName=90c0d1c8af782719d9f8d037e72c2a12/B, priority=12, startTime=1730983304549; duration=0sec 2024-11-07T12:41:44,681 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:41:44,681 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 90c0d1c8af782719d9f8d037e72c2a12:B 2024-11-07T12:41:44,681 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:41:44,681 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:44,681 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:44,682 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:44,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:44,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42994 deadline: 1730983364670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:44,683 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:44,683 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:44,683 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:44,684 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:41:44,684 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 90c0d1c8af782719d9f8d037e72c2a12/C is initiating minor compaction (all files) 2024-11-07T12:41:44,684 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 90c0d1c8af782719d9f8d037e72c2a12/C in TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:44,684 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/47ce9bc6ffe44d3eb1ace53f8d626150, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/c50b605f280044f59c4ae0341ced5511, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/191f690638bf42758c47af0fbc73bc03, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/8502002f71db4430a42f696609efdac6] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp, totalSize=44.6 K 2024-11-07T12:41:44,684 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 47ce9bc6ffe44d3eb1ace53f8d626150, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1730983299102 2024-11-07T12:41:44,685 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting c50b605f280044f59c4ae0341ced5511, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1730983299248 2024-11-07T12:41:44,686 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 191f690638bf42758c47af0fbc73bc03, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1730983301527 2024-11-07T12:41:44,686 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 8502002f71db4430a42f696609efdac6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=71, earliestPutTs=1730983303833 2024-11-07T12:41:44,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:44,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43028 deadline: 1730983364676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:44,696 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:44,696 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:44,697 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:44,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:44,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43062 deadline: 1730983364679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:44,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:44,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43000 deadline: 1730983364681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:44,706 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:44,706 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:44,706 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:44,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:44,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:44,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:44,712 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 90c0d1c8af782719d9f8d037e72c2a12#C#compaction#114 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:41:44,713 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/1f5534c1ceae4975a971ebc01e171453 is 50, key is test_row_0/C:col10/1730983303833/Put/seqid=0 2024-11-07T12:41:44,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741956_1132 (size=4469) 2024-11-07T12:41:44,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741957_1133 (size=12154) 2024-11-07T12:41:44,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:44,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:44,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43038 deadline: 1730983364792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:44,801 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107a6cb6a6f326d41a6a01e48905a25ae8c_90c0d1c8af782719d9f8d037e72c2a12 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107a6cb6a6f326d41a6a01e48905a25ae8c_90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:41:44,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741958_1134 (size=12139) 2024-11-07T12:41:44,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/61dbde97014940568754ffb791b95fa3, store: [table=TestAcidGuarantees family=A region=90c0d1c8af782719d9f8d037e72c2a12] 2024-11-07T12:41:44,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:44,804 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:44,804 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:44,804 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:44,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/61dbde97014940568754ffb791b95fa3 is 175, key is test_row_0/A:col10/1730983303950/Put/seqid=0 2024-11-07T12:41:44,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42994 deadline: 1730983364792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:44,805 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:44,805 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:44,805 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:44,817 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/1f5534c1ceae4975a971ebc01e171453 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/1f5534c1ceae4975a971ebc01e171453 2024-11-07T12:41:44,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:44,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43028 deadline: 1730983364804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:44,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:44,820 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:44,820 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:44,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43062 deadline: 1730983364813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:44,820 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:44,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:44,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:44,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:44,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:44,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43000 deadline: 1730983364813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:44,822 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:44,822 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:44,822 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:44,824 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 90c0d1c8af782719d9f8d037e72c2a12/C of 90c0d1c8af782719d9f8d037e72c2a12 into 1f5534c1ceae4975a971ebc01e171453(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:41:44,824 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 90c0d1c8af782719d9f8d037e72c2a12: 2024-11-07T12:41:44,825 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., storeName=90c0d1c8af782719d9f8d037e72c2a12/C, priority=12, startTime=1730983304549; duration=0sec 2024-11-07T12:41:44,825 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:41:44,825 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 90c0d1c8af782719d9f8d037e72c2a12:C 2024-11-07T12:41:44,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741959_1135 (size=30955) 2024-11-07T12:41:44,860 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=108, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/61dbde97014940568754ffb791b95fa3 2024-11-07T12:41:44,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/9758b955b00946f5bf71155e134ae797 is 50, key is test_row_0/B:col10/1730983303950/Put/seqid=0 2024-11-07T12:41:44,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741960_1136 (size=12001) 2024-11-07T12:41:44,942 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/9758b955b00946f5bf71155e134ae797 2024-11-07T12:41:44,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/66c2ccad37e34d83b3fef5712de17085 is 50, key is test_row_0/C:col10/1730983303950/Put/seqid=0 2024-11-07T12:41:45,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:45,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43038 deadline: 1730983365013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:45,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:45,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42994 deadline: 1730983365013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:45,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:45,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:45,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:45,029 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:45,029 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:45,029 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:45,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:45,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43062 deadline: 1730983365032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:45,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:45,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43028 deadline: 1730983365032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:45,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:45,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43000 deadline: 1730983365033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:45,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:45,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:45,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:45,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741961_1137 (size=12001) 2024-11-07T12:41:45,037 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:45,037 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:45,037 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:45,037 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:45,037 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:45,037 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:45,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-07T12:41:45,149 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 90c0d1c8af782719d9f8d037e72c2a12#A#compaction#112 average throughput is 0.04 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:41:45,149 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/821b1f1bee6241a99cf8c7bb1e2a7ed6 is 175, key is test_row_0/A:col10/1730983303833/Put/seqid=0 2024-11-07T12:41:45,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741962_1138 (size=31093) 2024-11-07T12:41:45,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:45,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42994 deadline: 1730983365333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:45,336 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:45,336 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:45,336 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:45,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:45,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43038 deadline: 1730983365337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:45,344 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:45,344 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:45,344 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:45,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:45,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43000 deadline: 1730983365343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:45,346 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:45,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:45,346 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:45,346 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:45,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43062 deadline: 1730983365343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:45,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:45,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:45,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43028 deadline: 1730983365343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:45,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:45,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:45,347 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:45,347 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:45,347 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:45,438 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/66c2ccad37e34d83b3fef5712de17085 2024-11-07T12:41:45,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/61dbde97014940568754ffb791b95fa3 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/61dbde97014940568754ffb791b95fa3 2024-11-07T12:41:45,479 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/61dbde97014940568754ffb791b95fa3, entries=150, sequenceid=108, filesize=30.2 K 2024-11-07T12:41:45,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/9758b955b00946f5bf71155e134ae797 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/9758b955b00946f5bf71155e134ae797 2024-11-07T12:41:45,491 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/9758b955b00946f5bf71155e134ae797, entries=150, sequenceid=108, filesize=11.7 K 2024-11-07T12:41:45,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/66c2ccad37e34d83b3fef5712de17085 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/66c2ccad37e34d83b3fef5712de17085 2024-11-07T12:41:45,507 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/66c2ccad37e34d83b3fef5712de17085, entries=150, sequenceid=108, filesize=11.7 K 2024-11-07T12:41:45,509 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 90c0d1c8af782719d9f8d037e72c2a12 in 945ms, sequenceid=108, compaction requested=false 2024-11-07T12:41:45,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2603): Flush status journal for 90c0d1c8af782719d9f8d037e72c2a12: 2024-11-07T12:41:45,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:45,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=37 2024-11-07T12:41:45,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4169): Remote procedure done, pid=37 2024-11-07T12:41:45,525 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-07T12:41:45,525 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5750 sec 2024-11-07T12:41:45,530 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees in 1.5940 sec 2024-11-07T12:41:45,575 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/821b1f1bee6241a99cf8c7bb1e2a7ed6 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/821b1f1bee6241a99cf8c7bb1e2a7ed6 2024-11-07T12:41:45,582 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 90c0d1c8af782719d9f8d037e72c2a12/A of 90c0d1c8af782719d9f8d037e72c2a12 into 821b1f1bee6241a99cf8c7bb1e2a7ed6(size=30.4 K), total size for store is 60.6 K. This selection was in queue for 0sec, and took 1sec to execute. 2024-11-07T12:41:45,582 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 90c0d1c8af782719d9f8d037e72c2a12: 2024-11-07T12:41:45,582 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., storeName=90c0d1c8af782719d9f8d037e72c2a12/A, priority=12, startTime=1730983304549; duration=1sec 2024-11-07T12:41:45,583 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:41:45,583 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 90c0d1c8af782719d9f8d037e72c2a12:A 2024-11-07T12:41:45,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:41:45,844 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 90c0d1c8af782719d9f8d037e72c2a12 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-07T12:41:45,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 90c0d1c8af782719d9f8d037e72c2a12, store=A 2024-11-07T12:41:45,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:45,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 90c0d1c8af782719d9f8d037e72c2a12, store=B 2024-11-07T12:41:45,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:45,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 90c0d1c8af782719d9f8d037e72c2a12, store=C 2024-11-07T12:41:45,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:45,878 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107ba38f17ddd664ffd815eace5964e408e_90c0d1c8af782719d9f8d037e72c2a12 is 50, key is test_row_0/A:col10/1730983304643/Put/seqid=0 2024-11-07T12:41:45,891 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1730983305864, max timestamp=1730983305887Num uniques 859; 2024-11-07T12:41:45,891 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-07T12:41:45,893 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1730983305864, max timestamp=1730983305887Num uniques 859; 2024-11-07T12:41:45,894 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-07T12:41:45,894 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1730983305864, max timestamp=1730983305887Num uniques 859; 2024-11-07T12:41:45,894 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-07T12:41:45,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741963_1139 (size=14594) 2024-11-07T12:41:45,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:45,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43062 deadline: 1730983365948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:45,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:45,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:45,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:45,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43000 deadline: 1730983365950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:45,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:45,959 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:45,959 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:45,959 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:45,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:45,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43028 deadline: 1730983365953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:45,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:45,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:45,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:45,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:45,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42994 deadline: 1730983365957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:45,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:45,965 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:45,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43038 deadline: 1730983365958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:45,965 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:45,965 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:45,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:45,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:45,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:46,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:46,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43062 deadline: 1730983366062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:46,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:46,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:46,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:46,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:46,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43000 deadline: 1730983366062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:46,072 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:46,072 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:46,072 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:46,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:46,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42994 deadline: 1730983366071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:46,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:46,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43028 deadline: 1730983366071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:46,080 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:46,080 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:46,080 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:46,080 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:46,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:46,080 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:46,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:46,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43038 deadline: 1730983366072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:46,084 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:46,084 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:46,084 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:46,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-07T12:41:46,085 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:41:46,086 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:41:46,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-11-07T12:41:46,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-07T12:41:46,089 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:41:46,092 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:41:46,092 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:41:46,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-07T12:41:46,245 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-07T12:41:46,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:46,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. as already flushing 2024-11-07T12:41:46,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:46,249 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:46,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:46,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:46,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:46,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43000 deadline: 1730983366282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:46,286 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:46,286 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:46,286 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:46,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:46,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43062 deadline: 1730983366283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:46,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:46,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:46,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:46,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:46,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42994 deadline: 1730983366291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:46,301 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:46,301 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:46,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:46,301 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:46,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43028 deadline: 1730983366292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:46,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:46,302 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:46,302 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:46,302 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:46,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43038 deadline: 1730983366292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:46,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:46,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:46,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:46,319 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:46,329 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107ba38f17ddd664ffd815eace5964e408e_90c0d1c8af782719d9f8d037e72c2a12 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107ba38f17ddd664ffd815eace5964e408e_90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:41:46,330 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/d4ea87bc8bcd475fbd43e3f8e7fd4368, store: [table=TestAcidGuarantees family=A region=90c0d1c8af782719d9f8d037e72c2a12] 2024-11-07T12:41:46,331 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/d4ea87bc8bcd475fbd43e3f8e7fd4368 is 175, key is test_row_0/A:col10/1730983304643/Put/seqid=0 2024-11-07T12:41:46,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741964_1140 (size=39549) 2024-11-07T12:41:46,343 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=122, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/d4ea87bc8bcd475fbd43e3f8e7fd4368 2024-11-07T12:41:46,355 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/a4dc22bb78734c9ca5ef51d8372b00ef is 50, key is test_row_0/B:col10/1730983304643/Put/seqid=0 2024-11-07T12:41:46,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741965_1141 (size=12001) 2024-11-07T12:41:46,370 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/a4dc22bb78734c9ca5ef51d8372b00ef 2024-11-07T12:41:46,385 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/aacbd0d5d673448e9a8076c06af80705 is 50, key is test_row_0/C:col10/1730983304643/Put/seqid=0 2024-11-07T12:41:46,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-07T12:41:46,408 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-07T12:41:46,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:46,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. as already flushing 2024-11-07T12:41:46,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:46,409 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:46,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:46,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:46,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741966_1142 (size=12001) 2024-11-07T12:41:46,447 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/aacbd0d5d673448e9a8076c06af80705 2024-11-07T12:41:46,454 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/d4ea87bc8bcd475fbd43e3f8e7fd4368 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/d4ea87bc8bcd475fbd43e3f8e7fd4368 2024-11-07T12:41:46,472 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/d4ea87bc8bcd475fbd43e3f8e7fd4368, entries=200, sequenceid=122, filesize=38.6 K 2024-11-07T12:41:46,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/a4dc22bb78734c9ca5ef51d8372b00ef as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/a4dc22bb78734c9ca5ef51d8372b00ef 2024-11-07T12:41:46,479 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/a4dc22bb78734c9ca5ef51d8372b00ef, entries=150, sequenceid=122, filesize=11.7 K 2024-11-07T12:41:46,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/aacbd0d5d673448e9a8076c06af80705 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/aacbd0d5d673448e9a8076c06af80705 2024-11-07T12:41:46,486 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/aacbd0d5d673448e9a8076c06af80705, entries=150, sequenceid=122, filesize=11.7 K 2024-11-07T12:41:46,487 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 90c0d1c8af782719d9f8d037e72c2a12 in 643ms, sequenceid=122, compaction requested=true 2024-11-07T12:41:46,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 90c0d1c8af782719d9f8d037e72c2a12: 2024-11-07T12:41:46,488 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:41:46,488 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 90c0d1c8af782719d9f8d037e72c2a12:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:41:46,488 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:41:46,488 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:41:46,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 90c0d1c8af782719d9f8d037e72c2a12:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:41:46,489 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101597 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:41:46,489 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): 90c0d1c8af782719d9f8d037e72c2a12/A is initiating minor compaction (all files) 2024-11-07T12:41:46,489 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 90c0d1c8af782719d9f8d037e72c2a12/A in TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:46,490 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/821b1f1bee6241a99cf8c7bb1e2a7ed6, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/61dbde97014940568754ffb791b95fa3, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/d4ea87bc8bcd475fbd43e3f8e7fd4368] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp, totalSize=99.2 K 2024-11-07T12:41:46,490 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:46,490 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. files: [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/821b1f1bee6241a99cf8c7bb1e2a7ed6, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/61dbde97014940568754ffb791b95fa3, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/d4ea87bc8bcd475fbd43e3f8e7fd4368] 2024-11-07T12:41:46,491 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:41:46,491 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 90c0d1c8af782719d9f8d037e72c2a12/B is initiating minor compaction (all files) 2024-11-07T12:41:46,491 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 90c0d1c8af782719d9f8d037e72c2a12/B in TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:46,491 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/ecdc018dd9194b0d8fe83793d52159a8, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/9758b955b00946f5bf71155e134ae797, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/a4dc22bb78734c9ca5ef51d8372b00ef] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp, totalSize=35.3 K 2024-11-07T12:41:46,492 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 821b1f1bee6241a99cf8c7bb1e2a7ed6, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=71, earliestPutTs=1730983303833 2024-11-07T12:41:46,492 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting ecdc018dd9194b0d8fe83793d52159a8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=71, earliestPutTs=1730983303833 2024-11-07T12:41:46,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:41:46,492 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 61dbde97014940568754ffb791b95fa3, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1730983303950 2024-11-07T12:41:46,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 90c0d1c8af782719d9f8d037e72c2a12:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:41:46,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:41:46,492 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 9758b955b00946f5bf71155e134ae797, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1730983303950 2024-11-07T12:41:46,493 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4ea87bc8bcd475fbd43e3f8e7fd4368, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1730983304643 2024-11-07T12:41:46,493 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting a4dc22bb78734c9ca5ef51d8372b00ef, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1730983304643 2024-11-07T12:41:46,510 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 90c0d1c8af782719d9f8d037e72c2a12#B#compaction#120 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:41:46,511 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/079dbb8b9b4d4b769d4525d014244aa8 is 50, key is test_row_0/B:col10/1730983304643/Put/seqid=0 2024-11-07T12:41:46,519 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=90c0d1c8af782719d9f8d037e72c2a12] 2024-11-07T12:41:46,526 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411075e48b855beea4f019b846dffe1e254c7_90c0d1c8af782719d9f8d037e72c2a12 store=[table=TestAcidGuarantees family=A region=90c0d1c8af782719d9f8d037e72c2a12] 2024-11-07T12:41:46,529 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411075e48b855beea4f019b846dffe1e254c7_90c0d1c8af782719d9f8d037e72c2a12, store=[table=TestAcidGuarantees family=A region=90c0d1c8af782719d9f8d037e72c2a12] 2024-11-07T12:41:46,529 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411075e48b855beea4f019b846dffe1e254c7_90c0d1c8af782719d9f8d037e72c2a12 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=90c0d1c8af782719d9f8d037e72c2a12] 2024-11-07T12:41:46,563 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-07T12:41:46,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:46,563 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing 90c0d1c8af782719d9f8d037e72c2a12 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-07T12:41:46,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 90c0d1c8af782719d9f8d037e72c2a12, store=A 2024-11-07T12:41:46,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:41:46,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 90c0d1c8af782719d9f8d037e72c2a12, store=B 2024-11-07T12:41:46,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:41:46,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 90c0d1c8af782719d9f8d037e72c2a12, store=C 2024-11-07T12:41:46,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:41:46,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741967_1143 (size=12241) 2024-11-07T12:41:46,601 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. as already flushing 2024-11-07T12:41:46,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:41:46,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:46,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:46,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43028 deadline: 1730983366612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:46,625 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:46,625 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:46,625 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:46,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42994 deadline: 1730983366612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:46,626 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:46,626 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:46,626 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:46,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:46,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43000 deadline: 1730983366620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:46,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:46,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:46,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:46,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:46,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43062 deadline: 1730983366625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:46,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:46,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43038 deadline: 1730983366625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:46,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:46,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:46,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:46,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:46,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:46,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:46,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741968_1144 (size=4469) 2024-11-07T12:41:46,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110739cb4a3e4ffb4fbfaace6ea97d7311ae_90c0d1c8af782719d9f8d037e72c2a12 is 50, key is test_row_0/A:col10/1730983305948/Put/seqid=0 2024-11-07T12:41:46,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741969_1145 (size=12304) 2024-11-07T12:41:46,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-07T12:41:46,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:46,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43000 deadline: 1730983366733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:46,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:46,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:46,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:46,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:46,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43038 deadline: 1730983366742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:46,750 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:46,750 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:46,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:46,750 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:46,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43062 deadline: 1730983366744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:46,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:46,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:46,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:46,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:46,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43000 deadline: 1730983366941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:46,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:46,947 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:46,947 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:46,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:46,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43038 deadline: 1730983366965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:46,966 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:46,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:46,966 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:46,966 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:46,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43062 deadline: 1730983366965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:46,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:46,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:46,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:46,991 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/079dbb8b9b4d4b769d4525d014244aa8 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/079dbb8b9b4d4b769d4525d014244aa8 2024-11-07T12:41:46,999 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 90c0d1c8af782719d9f8d037e72c2a12/B of 90c0d1c8af782719d9f8d037e72c2a12 into 079dbb8b9b4d4b769d4525d014244aa8(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:41:46,999 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 90c0d1c8af782719d9f8d037e72c2a12: 2024-11-07T12:41:46,999 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., storeName=90c0d1c8af782719d9f8d037e72c2a12/B, priority=13, startTime=1730983306488; duration=0sec 2024-11-07T12:41:46,999 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:41:46,999 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 90c0d1c8af782719d9f8d037e72c2a12:B 2024-11-07T12:41:46,999 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:41:47,002 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:41:47,002 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 90c0d1c8af782719d9f8d037e72c2a12/C is initiating minor compaction (all files) 2024-11-07T12:41:47,002 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 90c0d1c8af782719d9f8d037e72c2a12/C in TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:47,003 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/1f5534c1ceae4975a971ebc01e171453, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/66c2ccad37e34d83b3fef5712de17085, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/aacbd0d5d673448e9a8076c06af80705] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp, totalSize=35.3 K 2024-11-07T12:41:47,003 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f5534c1ceae4975a971ebc01e171453, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=71, earliestPutTs=1730983303833 2024-11-07T12:41:47,004 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 66c2ccad37e34d83b3fef5712de17085, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1730983303950 2024-11-07T12:41:47,004 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting aacbd0d5d673448e9a8076c06af80705, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1730983304643 2024-11-07T12:41:47,034 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 90c0d1c8af782719d9f8d037e72c2a12#C#compaction#123 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:41:47,035 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/00575456d4cf4570b87759b688c27c24 is 50, key is test_row_0/C:col10/1730983304643/Put/seqid=0 2024-11-07T12:41:47,040 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 90c0d1c8af782719d9f8d037e72c2a12#A#compaction#121 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:41:47,041 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/786576761519488483e13b8b2c1e6a81 is 175, key is test_row_0/A:col10/1730983304643/Put/seqid=0 2024-11-07T12:41:47,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741970_1146 (size=12241) 2024-11-07T12:41:47,050 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/00575456d4cf4570b87759b688c27c24 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/00575456d4cf4570b87759b688c27c24 2024-11-07T12:41:47,057 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 90c0d1c8af782719d9f8d037e72c2a12/C of 90c0d1c8af782719d9f8d037e72c2a12 into 00575456d4cf4570b87759b688c27c24(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:41:47,057 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 90c0d1c8af782719d9f8d037e72c2a12: 2024-11-07T12:41:47,057 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., storeName=90c0d1c8af782719d9f8d037e72c2a12/C, priority=13, startTime=1730983306492; duration=0sec 2024-11-07T12:41:47,057 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:41:47,057 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 90c0d1c8af782719d9f8d037e72c2a12:C 2024-11-07T12:41:47,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741971_1147 (size=31195) 2024-11-07T12:41:47,076 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/786576761519488483e13b8b2c1e6a81 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/786576761519488483e13b8b2c1e6a81 2024-11-07T12:41:47,087 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 90c0d1c8af782719d9f8d037e72c2a12/A of 90c0d1c8af782719d9f8d037e72c2a12 into 786576761519488483e13b8b2c1e6a81(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:41:47,087 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 90c0d1c8af782719d9f8d037e72c2a12: 2024-11-07T12:41:47,087 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., storeName=90c0d1c8af782719d9f8d037e72c2a12/A, priority=13, startTime=1730983306488; duration=0sec 2024-11-07T12:41:47,087 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:41:47,087 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 90c0d1c8af782719d9f8d037e72c2a12:A 2024-11-07T12:41:47,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:47,118 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110739cb4a3e4ffb4fbfaace6ea97d7311ae_90c0d1c8af782719d9f8d037e72c2a12 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110739cb4a3e4ffb4fbfaace6ea97d7311ae_90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:41:47,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/92f6310dadfc40a6a1c426a546001085, store: [table=TestAcidGuarantees family=A region=90c0d1c8af782719d9f8d037e72c2a12] 2024-11-07T12:41:47,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/92f6310dadfc40a6a1c426a546001085 is 175, key is test_row_0/A:col10/1730983305948/Put/seqid=0 2024-11-07T12:41:47,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:47,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43028 deadline: 1730983367132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:47,136 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:47,136 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:47,136 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:47,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:47,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42994 deadline: 1730983367142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:47,143 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:47,143 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:47,143 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:47,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741972_1148 (size=31105) 2024-11-07T12:41:47,167 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=158, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/92f6310dadfc40a6a1c426a546001085 2024-11-07T12:41:47,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/c3c6370ee41c401d9be1b718899b213e is 50, key is test_row_0/B:col10/1730983305948/Put/seqid=0 2024-11-07T12:41:47,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-07T12:41:47,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741973_1149 (size=12151) 2024-11-07T12:41:47,234 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/c3c6370ee41c401d9be1b718899b213e 2024-11-07T12:41:47,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:47,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43000 deadline: 1730983367251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:47,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/8818d0852d0544f3aa4b676dd78162d2 is 50, key is test_row_0/C:col10/1730983305948/Put/seqid=0 2024-11-07T12:41:47,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:47,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:47,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:47,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:47,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43062 deadline: 1730983367277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:47,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:47,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43038 deadline: 1730983367276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:47,279 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:47,279 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:47,279 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:47,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:47,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:47,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:47,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741974_1150 (size=12151) 2024-11-07T12:41:47,308 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/8818d0852d0544f3aa4b676dd78162d2 2024-11-07T12:41:47,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/92f6310dadfc40a6a1c426a546001085 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/92f6310dadfc40a6a1c426a546001085 2024-11-07T12:41:47,354 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/92f6310dadfc40a6a1c426a546001085, entries=150, sequenceid=158, filesize=30.4 K 2024-11-07T12:41:47,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/c3c6370ee41c401d9be1b718899b213e as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/c3c6370ee41c401d9be1b718899b213e 2024-11-07T12:41:47,412 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/c3c6370ee41c401d9be1b718899b213e, entries=150, sequenceid=158, filesize=11.9 K 2024-11-07T12:41:47,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/8818d0852d0544f3aa4b676dd78162d2 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/8818d0852d0544f3aa4b676dd78162d2 2024-11-07T12:41:47,428 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/8818d0852d0544f3aa4b676dd78162d2, entries=150, sequenceid=158, filesize=11.9 K 2024-11-07T12:41:47,437 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 90c0d1c8af782719d9f8d037e72c2a12 in 874ms, sequenceid=158, compaction requested=false 2024-11-07T12:41:47,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for 90c0d1c8af782719d9f8d037e72c2a12: 2024-11-07T12:41:47,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:47,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-07T12:41:47,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-07T12:41:47,453 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-07T12:41:47,453 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3530 sec 2024-11-07T12:41:47,466 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 1.3790 sec 2024-11-07T12:41:47,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:41:47,781 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 90c0d1c8af782719d9f8d037e72c2a12 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-07T12:41:47,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 90c0d1c8af782719d9f8d037e72c2a12, store=A 2024-11-07T12:41:47,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:47,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 90c0d1c8af782719d9f8d037e72c2a12, store=B 2024-11-07T12:41:47,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:47,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 90c0d1c8af782719d9f8d037e72c2a12, store=C 2024-11-07T12:41:47,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:47,804 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107382620a1e2f0459cad2f52342acb46ac_90c0d1c8af782719d9f8d037e72c2a12 is 50, key is test_row_0/A:col10/1730983306620/Put/seqid=0 2024-11-07T12:41:47,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741975_1151 (size=14794) 2024-11-07T12:41:47,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:47,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43062 deadline: 1730983367879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:47,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:47,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:47,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:47,891 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1730983307859, max timestamp=1730983307884Num uniques 859; 2024-11-07T12:41:47,891 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-07T12:41:47,893 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1730983307859, max timestamp=1730983307884Num uniques 859; 2024-11-07T12:41:47,894 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-07T12:41:47,895 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1730983307859, max timestamp=1730983307884Num uniques 859; 2024-11-07T12:41:47,895 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-07T12:41:48,010 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:48,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43038 deadline: 1730983367996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:48,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:48,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:48,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:48,011 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:48,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43062 deadline: 1730983368001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:48,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:48,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:48,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:48,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:48,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43000 deadline: 1730983368007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:48,016 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:48,016 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:48,016 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:48,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:48,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43038 deadline: 1730983368122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:48,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:48,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:48,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:48,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:48,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43000 deadline: 1730983368122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:48,132 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:48,132 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:48,132 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:48,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:48,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43028 deadline: 1730983368152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:48,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:48,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42994 deadline: 1730983368152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:48,156 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to db9ad1cb6cf9,34293,1730983260217 for 'test_row_1' in 90c0d1c8af782719d9f8d037e72c2a12 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:41:48,156 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to db9ad1cb6cf9,34293,1730983260217 for 'test_row_0' in 90c0d1c8af782719d9f8d037e72c2a12 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:41:48,157 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:48,157 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:48,157 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:48,157 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:48,157 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:48,157 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:48,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:48,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43062 deadline: 1730983368223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:48,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:48,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:48,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:48,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-07T12:41:48,232 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:41:48,233 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:41:48,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-07T12:41:48,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-07T12:41:48,235 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:41:48,236 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:41:48,236 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:41:48,247 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:48,252 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107382620a1e2f0459cad2f52342acb46ac_90c0d1c8af782719d9f8d037e72c2a12 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107382620a1e2f0459cad2f52342acb46ac_90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:41:48,253 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/61ca6416d8d54eb4bcf3f4a893bcb846, store: [table=TestAcidGuarantees family=A region=90c0d1c8af782719d9f8d037e72c2a12] 2024-11-07T12:41:48,253 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/61ca6416d8d54eb4bcf3f4a893bcb846 is 175, key is test_row_0/A:col10/1730983306620/Put/seqid=0 2024-11-07T12:41:48,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741976_1152 (size=39749) 2024-11-07T12:41:48,261 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=172, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/61ca6416d8d54eb4bcf3f4a893bcb846 2024-11-07T12:41:48,272 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/afdd0ce03813446793ef00b990070de2 is 50, key is test_row_0/B:col10/1730983306620/Put/seqid=0 2024-11-07T12:41:48,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741977_1153 (size=12151) 2024-11-07T12:41:48,280 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/afdd0ce03813446793ef00b990070de2 2024-11-07T12:41:48,290 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/f1fa25483a81465d80dc8c1d05d39e4d is 50, key is test_row_0/C:col10/1730983306620/Put/seqid=0 2024-11-07T12:41:48,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741978_1154 (size=12151) 2024-11-07T12:41:48,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:48,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43038 deadline: 1730983368332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:48,335 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/f1fa25483a81465d80dc8c1d05d39e4d 2024-11-07T12:41:48,335 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:48,335 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:48,335 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:48,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-07T12:41:48,343 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/61ca6416d8d54eb4bcf3f4a893bcb846 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/61ca6416d8d54eb4bcf3f4a893bcb846 2024-11-07T12:41:48,349 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/61ca6416d8d54eb4bcf3f4a893bcb846, entries=200, sequenceid=172, filesize=38.8 K 2024-11-07T12:41:48,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:48,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43000 deadline: 1730983368341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:48,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:48,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:48,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:48,353 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/afdd0ce03813446793ef00b990070de2 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/afdd0ce03813446793ef00b990070de2 2024-11-07T12:41:48,359 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/afdd0ce03813446793ef00b990070de2, entries=150, sequenceid=172, filesize=11.9 K 2024-11-07T12:41:48,360 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/f1fa25483a81465d80dc8c1d05d39e4d as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/f1fa25483a81465d80dc8c1d05d39e4d 2024-11-07T12:41:48,364 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/f1fa25483a81465d80dc8c1d05d39e4d, entries=150, sequenceid=172, filesize=11.9 K 2024-11-07T12:41:48,365 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 90c0d1c8af782719d9f8d037e72c2a12 in 584ms, sequenceid=172, compaction requested=true 2024-11-07T12:41:48,366 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 90c0d1c8af782719d9f8d037e72c2a12: 2024-11-07T12:41:48,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 90c0d1c8af782719d9f8d037e72c2a12:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:41:48,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:41:48,366 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:41:48,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 90c0d1c8af782719d9f8d037e72c2a12:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:41:48,366 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:41:48,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:41:48,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 90c0d1c8af782719d9f8d037e72c2a12:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:41:48,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:41:48,367 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:41:48,367 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:41:48,367 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): 90c0d1c8af782719d9f8d037e72c2a12/A is initiating minor compaction (all files) 2024-11-07T12:41:48,367 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 90c0d1c8af782719d9f8d037e72c2a12/B is initiating minor compaction (all files) 2024-11-07T12:41:48,367 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 90c0d1c8af782719d9f8d037e72c2a12/A in TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:48,367 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 90c0d1c8af782719d9f8d037e72c2a12/B in TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:48,367 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/786576761519488483e13b8b2c1e6a81, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/92f6310dadfc40a6a1c426a546001085, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/61ca6416d8d54eb4bcf3f4a893bcb846] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp, totalSize=99.7 K 2024-11-07T12:41:48,368 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/079dbb8b9b4d4b769d4525d014244aa8, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/c3c6370ee41c401d9be1b718899b213e, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/afdd0ce03813446793ef00b990070de2] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp, totalSize=35.7 K 2024-11-07T12:41:48,368 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:48,368 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. files: [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/786576761519488483e13b8b2c1e6a81, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/92f6310dadfc40a6a1c426a546001085, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/61ca6416d8d54eb4bcf3f4a893bcb846] 2024-11-07T12:41:48,368 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 079dbb8b9b4d4b769d4525d014244aa8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1730983304643 2024-11-07T12:41:48,368 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 786576761519488483e13b8b2c1e6a81, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1730983304643 2024-11-07T12:41:48,369 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92f6310dadfc40a6a1c426a546001085, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1730983305948 2024-11-07T12:41:48,369 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting c3c6370ee41c401d9be1b718899b213e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1730983305948 2024-11-07T12:41:48,369 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 61ca6416d8d54eb4bcf3f4a893bcb846, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1730983306608 2024-11-07T12:41:48,369 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting afdd0ce03813446793ef00b990070de2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1730983306608 2024-11-07T12:41:48,389 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-07T12:41:48,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:48,389 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 90c0d1c8af782719d9f8d037e72c2a12 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-07T12:41:48,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 90c0d1c8af782719d9f8d037e72c2a12, store=A 2024-11-07T12:41:48,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:41:48,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 90c0d1c8af782719d9f8d037e72c2a12, store=B 2024-11-07T12:41:48,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:41:48,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 90c0d1c8af782719d9f8d037e72c2a12, store=C 2024-11-07T12:41:48,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:41:48,391 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 90c0d1c8af782719d9f8d037e72c2a12#B#compaction#129 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:41:48,392 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/3a317eea129b4cb4af883a40293ff241 is 50, key is test_row_0/B:col10/1730983306620/Put/seqid=0 2024-11-07T12:41:48,398 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=90c0d1c8af782719d9f8d037e72c2a12] 2024-11-07T12:41:48,416 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411076bad7ac9a26e4c048ba7b1b527444e88_90c0d1c8af782719d9f8d037e72c2a12 store=[table=TestAcidGuarantees family=A region=90c0d1c8af782719d9f8d037e72c2a12] 2024-11-07T12:41:48,419 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411076bad7ac9a26e4c048ba7b1b527444e88_90c0d1c8af782719d9f8d037e72c2a12, store=[table=TestAcidGuarantees family=A region=90c0d1c8af782719d9f8d037e72c2a12] 2024-11-07T12:41:48,419 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411076bad7ac9a26e4c048ba7b1b527444e88_90c0d1c8af782719d9f8d037e72c2a12 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=90c0d1c8af782719d9f8d037e72c2a12] 2024-11-07T12:41:48,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110718ec6594ef684861afdc6577a0f829e3_90c0d1c8af782719d9f8d037e72c2a12 is 50, key is test_row_0/A:col10/1730983307979/Put/seqid=0 2024-11-07T12:41:48,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741979_1155 (size=12493) 2024-11-07T12:41:48,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741980_1156 (size=4469) 2024-11-07T12:41:48,493 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 90c0d1c8af782719d9f8d037e72c2a12#A#compaction#130 average throughput is 0.26 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:41:48,493 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/ecd77c8d37cb4d3bbd6a358f36a947b5 is 175, key is test_row_0/A:col10/1730983306620/Put/seqid=0 2024-11-07T12:41:48,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741981_1157 (size=12304) 2024-11-07T12:41:48,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:48,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741982_1158 (size=31447) 2024-11-07T12:41:48,523 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110718ec6594ef684861afdc6577a0f829e3_90c0d1c8af782719d9f8d037e72c2a12 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110718ec6594ef684861afdc6577a0f829e3_90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:41:48,531 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/ecd77c8d37cb4d3bbd6a358f36a947b5 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/ecd77c8d37cb4d3bbd6a358f36a947b5 2024-11-07T12:41:48,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/a9000c06a877406c981aee490c136ba8, store: [table=TestAcidGuarantees family=A region=90c0d1c8af782719d9f8d037e72c2a12] 2024-11-07T12:41:48,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/a9000c06a877406c981aee490c136ba8 is 175, key is test_row_0/A:col10/1730983307979/Put/seqid=0 2024-11-07T12:41:48,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:41:48,540 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. as already flushing 2024-11-07T12:41:48,546 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 90c0d1c8af782719d9f8d037e72c2a12/A of 90c0d1c8af782719d9f8d037e72c2a12 into ecd77c8d37cb4d3bbd6a358f36a947b5(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:41:48,546 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 90c0d1c8af782719d9f8d037e72c2a12: 2024-11-07T12:41:48,546 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., storeName=90c0d1c8af782719d9f8d037e72c2a12/A, priority=13, startTime=1730983308366; duration=0sec 2024-11-07T12:41:48,546 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:41:48,547 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 90c0d1c8af782719d9f8d037e72c2a12:A 2024-11-07T12:41:48,547 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:41:48,548 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:41:48,548 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): 90c0d1c8af782719d9f8d037e72c2a12/C is initiating minor compaction (all files) 2024-11-07T12:41:48,548 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 90c0d1c8af782719d9f8d037e72c2a12/C in TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:48,548 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/00575456d4cf4570b87759b688c27c24, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/8818d0852d0544f3aa4b676dd78162d2, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/f1fa25483a81465d80dc8c1d05d39e4d] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp, totalSize=35.7 K 2024-11-07T12:41:48,549 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00575456d4cf4570b87759b688c27c24, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1730983304643 2024-11-07T12:41:48,550 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8818d0852d0544f3aa4b676dd78162d2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1730983305948 2024-11-07T12:41:48,551 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1fa25483a81465d80dc8c1d05d39e4d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1730983306608 2024-11-07T12:41:48,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-07T12:41:48,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741983_1159 (size=31105) 2024-11-07T12:41:48,569 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 90c0d1c8af782719d9f8d037e72c2a12#C#compaction#132 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:41:48,569 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=207, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/a9000c06a877406c981aee490c136ba8 2024-11-07T12:41:48,570 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/f29b0ee4db864c799f8cf958f2e894e0 is 50, key is test_row_0/C:col10/1730983306620/Put/seqid=0 2024-11-07T12:41:48,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/6218b4f9350e4fa8a230d63194b4d469 is 50, key is test_row_0/B:col10/1730983307979/Put/seqid=0 2024-11-07T12:41:48,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:48,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43062 deadline: 1730983368617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:48,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:48,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:48,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:48,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741984_1160 (size=12493) 2024-11-07T12:41:48,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741985_1161 (size=12151) 2024-11-07T12:41:48,643 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/6218b4f9350e4fa8a230d63194b4d469 2024-11-07T12:41:48,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:48,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43038 deadline: 1730983368642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:48,647 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:48,647 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:48,647 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:48,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/1abb5d6c5ca54a1abe404f1a3c0848e2 is 50, key is test_row_0/C:col10/1730983307979/Put/seqid=0 2024-11-07T12:41:48,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:48,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43000 deadline: 1730983368661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:48,667 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:48,667 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:48,667 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:48,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741986_1162 (size=12151) 2024-11-07T12:41:48,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:48,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43062 deadline: 1730983368731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:48,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:48,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:48,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:48,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-07T12:41:48,870 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/3a317eea129b4cb4af883a40293ff241 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/3a317eea129b4cb4af883a40293ff241 2024-11-07T12:41:48,885 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 90c0d1c8af782719d9f8d037e72c2a12/B of 90c0d1c8af782719d9f8d037e72c2a12 into 3a317eea129b4cb4af883a40293ff241(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:41:48,885 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 90c0d1c8af782719d9f8d037e72c2a12: 2024-11-07T12:41:48,885 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., storeName=90c0d1c8af782719d9f8d037e72c2a12/B, priority=13, startTime=1730983308366; duration=0sec 2024-11-07T12:41:48,885 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:41:48,885 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 90c0d1c8af782719d9f8d037e72c2a12:B 2024-11-07T12:41:48,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:48,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43062 deadline: 1730983368942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:48,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:48,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:48,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:49,040 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/f29b0ee4db864c799f8cf958f2e894e0 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/f29b0ee4db864c799f8cf958f2e894e0 2024-11-07T12:41:49,048 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 90c0d1c8af782719d9f8d037e72c2a12/C of 90c0d1c8af782719d9f8d037e72c2a12 into f29b0ee4db864c799f8cf958f2e894e0(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:41:49,048 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 90c0d1c8af782719d9f8d037e72c2a12: 2024-11-07T12:41:49,048 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., storeName=90c0d1c8af782719d9f8d037e72c2a12/C, priority=13, startTime=1730983308366; duration=0sec 2024-11-07T12:41:49,048 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:41:49,048 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 90c0d1c8af782719d9f8d037e72c2a12:C 2024-11-07T12:41:49,127 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/1abb5d6c5ca54a1abe404f1a3c0848e2 2024-11-07T12:41:49,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/a9000c06a877406c981aee490c136ba8 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/a9000c06a877406c981aee490c136ba8 2024-11-07T12:41:49,142 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/a9000c06a877406c981aee490c136ba8, entries=150, sequenceid=207, filesize=30.4 K 2024-11-07T12:41:49,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/6218b4f9350e4fa8a230d63194b4d469 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/6218b4f9350e4fa8a230d63194b4d469 2024-11-07T12:41:49,150 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/6218b4f9350e4fa8a230d63194b4d469, entries=150, sequenceid=207, filesize=11.9 K 2024-11-07T12:41:49,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/1abb5d6c5ca54a1abe404f1a3c0848e2 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/1abb5d6c5ca54a1abe404f1a3c0848e2 2024-11-07T12:41:49,158 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/1abb5d6c5ca54a1abe404f1a3c0848e2, entries=150, sequenceid=207, filesize=11.9 K 2024-11-07T12:41:49,160 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 90c0d1c8af782719d9f8d037e72c2a12 in 771ms, sequenceid=207, compaction requested=false 2024-11-07T12:41:49,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 90c0d1c8af782719d9f8d037e72c2a12: 2024-11-07T12:41:49,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:49,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-07T12:41:49,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-07T12:41:49,170 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-07T12:41:49,170 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 926 msec 2024-11-07T12:41:49,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:41:49,171 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 90c0d1c8af782719d9f8d037e72c2a12 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-07T12:41:49,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 90c0d1c8af782719d9f8d037e72c2a12, store=A 2024-11-07T12:41:49,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:49,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 90c0d1c8af782719d9f8d037e72c2a12, store=B 2024-11-07T12:41:49,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:49,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 90c0d1c8af782719d9f8d037e72c2a12, store=C 2024-11-07T12:41:49,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:49,174 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 939 msec 2024-11-07T12:41:49,203 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110714ee5edac1414cc3af3e6baed8bb846f_90c0d1c8af782719d9f8d037e72c2a12 is 50, key is test_row_0/A:col10/1730983308586/Put/seqid=0 2024-11-07T12:41:49,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:49,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43038 deadline: 1730983369237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:49,240 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:49,240 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:49,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43000 deadline: 1730983369239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:49,240 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:49,240 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:49,241 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:49,241 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:49,241 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:49,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:49,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43062 deadline: 1730983369252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:49,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:49,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:49,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:49,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741987_1163 (size=14794) 2024-11-07T12:41:49,264 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:49,269 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110714ee5edac1414cc3af3e6baed8bb846f_90c0d1c8af782719d9f8d037e72c2a12 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110714ee5edac1414cc3af3e6baed8bb846f_90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:41:49,273 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/4a5602684f7342b2a0a32d26ce121c3a, store: [table=TestAcidGuarantees family=A region=90c0d1c8af782719d9f8d037e72c2a12] 2024-11-07T12:41:49,273 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/4a5602684f7342b2a0a32d26ce121c3a is 175, key is test_row_0/A:col10/1730983308586/Put/seqid=0 2024-11-07T12:41:49,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741988_1164 (size=39749) 2024-11-07T12:41:49,284 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=222, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/4a5602684f7342b2a0a32d26ce121c3a 2024-11-07T12:41:49,299 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/ae85411afa064738b4bccd784b031c7a is 50, key is test_row_0/B:col10/1730983308586/Put/seqid=0 2024-11-07T12:41:49,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741989_1165 (size=12151) 2024-11-07T12:41:49,317 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/ae85411afa064738b4bccd784b031c7a 2024-11-07T12:41:49,343 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/72c8293d7d8c4e8aa4e496516e8eceb4 is 50, key is test_row_0/C:col10/1730983308586/Put/seqid=0 2024-11-07T12:41:49,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:49,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43038 deadline: 1730983369342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:49,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:49,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:49,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:49,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:49,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43000 deadline: 1730983369351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:49,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:49,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:49,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:49,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-07T12:41:49,373 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:41:49,373 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:41:49,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-07T12:41:49,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-07T12:41:49,380 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:41:49,383 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:41:49,383 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:41:49,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741990_1166 (size=12151) 2024-11-07T12:41:49,403 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/72c8293d7d8c4e8aa4e496516e8eceb4 2024-11-07T12:41:49,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/4a5602684f7342b2a0a32d26ce121c3a as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/4a5602684f7342b2a0a32d26ce121c3a 2024-11-07T12:41:49,415 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/4a5602684f7342b2a0a32d26ce121c3a, entries=200, sequenceid=222, filesize=38.8 K 2024-11-07T12:41:49,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/ae85411afa064738b4bccd784b031c7a as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/ae85411afa064738b4bccd784b031c7a 2024-11-07T12:41:49,422 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/ae85411afa064738b4bccd784b031c7a, entries=150, sequenceid=222, filesize=11.9 K 2024-11-07T12:41:49,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/72c8293d7d8c4e8aa4e496516e8eceb4 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/72c8293d7d8c4e8aa4e496516e8eceb4 2024-11-07T12:41:49,431 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/72c8293d7d8c4e8aa4e496516e8eceb4, entries=150, sequenceid=222, filesize=11.9 K 2024-11-07T12:41:49,432 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 90c0d1c8af782719d9f8d037e72c2a12 in 261ms, sequenceid=222, compaction requested=true 2024-11-07T12:41:49,432 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 90c0d1c8af782719d9f8d037e72c2a12: 2024-11-07T12:41:49,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 90c0d1c8af782719d9f8d037e72c2a12:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:41:49,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:41:49,433 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:41:49,433 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:41:49,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 90c0d1c8af782719d9f8d037e72c2a12:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:41:49,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:41:49,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 90c0d1c8af782719d9f8d037e72c2a12:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:41:49,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:41:49,434 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102301 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:41:49,434 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): 90c0d1c8af782719d9f8d037e72c2a12/A is initiating minor compaction (all files) 2024-11-07T12:41:49,434 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 90c0d1c8af782719d9f8d037e72c2a12/A in TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:49,434 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/ecd77c8d37cb4d3bbd6a358f36a947b5, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/a9000c06a877406c981aee490c136ba8, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/4a5602684f7342b2a0a32d26ce121c3a] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp, totalSize=99.9 K 2024-11-07T12:41:49,434 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:49,434 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. files: [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/ecd77c8d37cb4d3bbd6a358f36a947b5, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/a9000c06a877406c981aee490c136ba8, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/4a5602684f7342b2a0a32d26ce121c3a] 2024-11-07T12:41:49,435 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:41:49,435 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 90c0d1c8af782719d9f8d037e72c2a12/B is initiating minor compaction (all files) 2024-11-07T12:41:49,435 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 90c0d1c8af782719d9f8d037e72c2a12/B in TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:49,436 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/3a317eea129b4cb4af883a40293ff241, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/6218b4f9350e4fa8a230d63194b4d469, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/ae85411afa064738b4bccd784b031c7a] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp, totalSize=35.9 K 2024-11-07T12:41:49,436 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting ecd77c8d37cb4d3bbd6a358f36a947b5, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1730983306608 2024-11-07T12:41:49,436 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a317eea129b4cb4af883a40293ff241, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1730983306608 2024-11-07T12:41:49,437 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 6218b4f9350e4fa8a230d63194b4d469, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1730983307877 2024-11-07T12:41:49,437 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9000c06a877406c981aee490c136ba8, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1730983307877 2024-11-07T12:41:49,438 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting ae85411afa064738b4bccd784b031c7a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1730983308573 2024-11-07T12:41:49,438 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a5602684f7342b2a0a32d26ce121c3a, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1730983308573 2024-11-07T12:41:49,460 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 90c0d1c8af782719d9f8d037e72c2a12#B#compaction#138 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:41:49,460 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/5684dac7279347ce9a65a3b1f952814d is 50, key is test_row_0/B:col10/1730983308586/Put/seqid=0 2024-11-07T12:41:49,473 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=90c0d1c8af782719d9f8d037e72c2a12] 2024-11-07T12:41:49,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-07T12:41:49,485 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411076d27f95a52cc4872b3ee70cf738a744f_90c0d1c8af782719d9f8d037e72c2a12 store=[table=TestAcidGuarantees family=A region=90c0d1c8af782719d9f8d037e72c2a12] 2024-11-07T12:41:49,488 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411076d27f95a52cc4872b3ee70cf738a744f_90c0d1c8af782719d9f8d037e72c2a12, store=[table=TestAcidGuarantees family=A region=90c0d1c8af782719d9f8d037e72c2a12] 2024-11-07T12:41:49,488 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411076d27f95a52cc4872b3ee70cf738a744f_90c0d1c8af782719d9f8d037e72c2a12 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=90c0d1c8af782719d9f8d037e72c2a12] 2024-11-07T12:41:49,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741991_1167 (size=12595) 2024-11-07T12:41:49,532 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/5684dac7279347ce9a65a3b1f952814d as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/5684dac7279347ce9a65a3b1f952814d 2024-11-07T12:41:49,538 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-07T12:41:49,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:49,539 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 90c0d1c8af782719d9f8d037e72c2a12 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-07T12:41:49,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 90c0d1c8af782719d9f8d037e72c2a12, store=A 2024-11-07T12:41:49,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:49,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 90c0d1c8af782719d9f8d037e72c2a12, store=B 2024-11-07T12:41:49,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:49,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 90c0d1c8af782719d9f8d037e72c2a12, store=C 2024-11-07T12:41:49,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:49,546 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 90c0d1c8af782719d9f8d037e72c2a12/B of 90c0d1c8af782719d9f8d037e72c2a12 into 5684dac7279347ce9a65a3b1f952814d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:41:49,546 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 90c0d1c8af782719d9f8d037e72c2a12: 2024-11-07T12:41:49,546 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., storeName=90c0d1c8af782719d9f8d037e72c2a12/B, priority=13, startTime=1730983309433; duration=0sec 2024-11-07T12:41:49,549 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:41:49,549 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 90c0d1c8af782719d9f8d037e72c2a12:B 2024-11-07T12:41:49,549 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:41:49,551 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:41:49,551 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 90c0d1c8af782719d9f8d037e72c2a12/C is initiating minor compaction (all files) 2024-11-07T12:41:49,551 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 90c0d1c8af782719d9f8d037e72c2a12/C in TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:49,551 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/f29b0ee4db864c799f8cf958f2e894e0, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/1abb5d6c5ca54a1abe404f1a3c0848e2, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/72c8293d7d8c4e8aa4e496516e8eceb4] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp, totalSize=35.9 K 2024-11-07T12:41:49,553 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting f29b0ee4db864c799f8cf958f2e894e0, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1730983306608 2024-11-07T12:41:49,555 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. as already flushing 2024-11-07T12:41:49,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:41:49,557 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 1abb5d6c5ca54a1abe404f1a3c0848e2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1730983307877 2024-11-07T12:41:49,557 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 72c8293d7d8c4e8aa4e496516e8eceb4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1730983308573 2024-11-07T12:41:49,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741992_1168 (size=4469) 2024-11-07T12:41:49,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107d0ecbadfdb5a4d80b85d426221ef9284_90c0d1c8af782719d9f8d037e72c2a12 is 50, key is test_row_0/A:col10/1730983309235/Put/seqid=0 2024-11-07T12:41:49,592 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 90c0d1c8af782719d9f8d037e72c2a12#C#compaction#141 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:41:49,592 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/fe26956dc6e14c88b9c9d52c23372a57 is 50, key is test_row_0/C:col10/1730983308586/Put/seqid=0 2024-11-07T12:41:49,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:49,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43000 deadline: 1730983369588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:49,596 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:49,596 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:49,596 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:49,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741993_1169 (size=12304) 2024-11-07T12:41:49,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:49,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:49,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43038 deadline: 1730983369595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:49,605 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107d0ecbadfdb5a4d80b85d426221ef9284_90c0d1c8af782719d9f8d037e72c2a12 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107d0ecbadfdb5a4d80b85d426221ef9284_90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:41:49,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:49,606 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:49,606 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:49,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/a0ee187b1da8408d9d73c194d72b4e01, store: [table=TestAcidGuarantees family=A region=90c0d1c8af782719d9f8d037e72c2a12] 2024-11-07T12:41:49,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/a0ee187b1da8408d9d73c194d72b4e01 is 175, key is test_row_0/A:col10/1730983309235/Put/seqid=0 2024-11-07T12:41:49,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741994_1170 (size=12595) 2024-11-07T12:41:49,628 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/fe26956dc6e14c88b9c9d52c23372a57 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/fe26956dc6e14c88b9c9d52c23372a57 2024-11-07T12:41:49,637 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 90c0d1c8af782719d9f8d037e72c2a12/C of 90c0d1c8af782719d9f8d037e72c2a12 into fe26956dc6e14c88b9c9d52c23372a57(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:41:49,637 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 90c0d1c8af782719d9f8d037e72c2a12: 2024-11-07T12:41:49,637 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., storeName=90c0d1c8af782719d9f8d037e72c2a12/C, priority=13, startTime=1730983309433; duration=0sec 2024-11-07T12:41:49,638 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:41:49,638 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 90c0d1c8af782719d9f8d037e72c2a12:C 2024-11-07T12:41:49,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741995_1171 (size=31105) 2024-11-07T12:41:49,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-07T12:41:49,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:49,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43000 deadline: 1730983369701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:49,704 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:49,704 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:49,704 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:49,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:49,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43038 deadline: 1730983369712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:49,719 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:49,719 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:49,719 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:49,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:49,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43062 deadline: 1730983369761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:49,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:49,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:49,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:49,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:49,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43000 deadline: 1730983369912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:49,919 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:49,919 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:49,919 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:49,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:49,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43038 deadline: 1730983369922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:49,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:49,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:49,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:49,966 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 90c0d1c8af782719d9f8d037e72c2a12#A#compaction#139 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:41:49,967 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/7757eeeeef414dd5a23bcc6569148a3c is 175, key is test_row_0/A:col10/1730983308586/Put/seqid=0 2024-11-07T12:41:49,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741996_1172 (size=31549) 2024-11-07T12:41:50,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-07T12:41:50,062 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=242, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/a0ee187b1da8408d9d73c194d72b4e01 2024-11-07T12:41:50,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/e0f21398f98f49a58406a171ebd94b93 is 50, key is test_row_0/B:col10/1730983309235/Put/seqid=0 2024-11-07T12:41:50,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741997_1173 (size=12151) 2024-11-07T12:41:50,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:50,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42994 deadline: 1730983370181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:50,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:50,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43028 deadline: 1730983370182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:50,183 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to db9ad1cb6cf9,34293,1730983260217 for 'test_row_1' in 90c0d1c8af782719d9f8d037e72c2a12 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4230 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:41:50,183 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to db9ad1cb6cf9,34293,1730983260217 for 'test_row_0' in 90c0d1c8af782719d9f8d037e72c2a12 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4226 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:41:50,184 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:50,184 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:50,184 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:50,184 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:50,184 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:50,184 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:50,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:50,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43000 deadline: 1730983370232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:50,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:50,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43038 deadline: 1730983370232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:50,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:50,241 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:50,241 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:50,241 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:50,241 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:50,241 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:50,387 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/7757eeeeef414dd5a23bcc6569148a3c as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/7757eeeeef414dd5a23bcc6569148a3c 2024-11-07T12:41:50,417 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 90c0d1c8af782719d9f8d037e72c2a12/A of 90c0d1c8af782719d9f8d037e72c2a12 into 7757eeeeef414dd5a23bcc6569148a3c(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:41:50,417 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 90c0d1c8af782719d9f8d037e72c2a12: 2024-11-07T12:41:50,417 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., storeName=90c0d1c8af782719d9f8d037e72c2a12/A, priority=13, startTime=1730983309433; duration=0sec 2024-11-07T12:41:50,417 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:41:50,417 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 90c0d1c8af782719d9f8d037e72c2a12:A 2024-11-07T12:41:50,488 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/e0f21398f98f49a58406a171ebd94b93 2024-11-07T12:41:50,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/76f0c164606340238fccfc5068df4d24 is 50, key is test_row_0/C:col10/1730983309235/Put/seqid=0 2024-11-07T12:41:50,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-07T12:41:50,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741998_1174 (size=12151) 2024-11-07T12:41:50,553 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/76f0c164606340238fccfc5068df4d24 2024-11-07T12:41:50,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/a0ee187b1da8408d9d73c194d72b4e01 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/a0ee187b1da8408d9d73c194d72b4e01 2024-11-07T12:41:50,567 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/a0ee187b1da8408d9d73c194d72b4e01, entries=150, sequenceid=242, filesize=30.4 K 2024-11-07T12:41:50,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/e0f21398f98f49a58406a171ebd94b93 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/e0f21398f98f49a58406a171ebd94b93 2024-11-07T12:41:50,583 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/e0f21398f98f49a58406a171ebd94b93, entries=150, sequenceid=242, filesize=11.9 K 2024-11-07T12:41:50,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/76f0c164606340238fccfc5068df4d24 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/76f0c164606340238fccfc5068df4d24 2024-11-07T12:41:50,594 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/76f0c164606340238fccfc5068df4d24, entries=150, sequenceid=242, filesize=11.9 K 2024-11-07T12:41:50,595 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 90c0d1c8af782719d9f8d037e72c2a12 in 1056ms, sequenceid=242, compaction requested=false 2024-11-07T12:41:50,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 90c0d1c8af782719d9f8d037e72c2a12: 2024-11-07T12:41:50,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:50,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-07T12:41:50,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-07T12:41:50,598 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-07T12:41:50,598 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2140 sec 2024-11-07T12:41:50,601 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.2260 sec 2024-11-07T12:41:50,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:41:50,753 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 90c0d1c8af782719d9f8d037e72c2a12 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-07T12:41:50,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 90c0d1c8af782719d9f8d037e72c2a12, store=A 2024-11-07T12:41:50,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:50,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 90c0d1c8af782719d9f8d037e72c2a12, store=B 2024-11-07T12:41:50,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:50,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 90c0d1c8af782719d9f8d037e72c2a12, store=C 2024-11-07T12:41:50,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:50,788 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107508e020ff1644ac6b7e1270d588a0614_90c0d1c8af782719d9f8d037e72c2a12 is 50, key is test_row_0/A:col10/1730983309586/Put/seqid=0 2024-11-07T12:41:50,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:50,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43000 deadline: 1730983370790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:50,794 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:50,795 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:50,795 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:50,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:50,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43062 deadline: 1730983370790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:50,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:50,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:50,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:50,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:50,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43038 deadline: 1730983370794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:50,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:50,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:50,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:50,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741999_1175 (size=14844) 2024-11-07T12:41:50,817 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:50,822 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107508e020ff1644ac6b7e1270d588a0614_90c0d1c8af782719d9f8d037e72c2a12 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107508e020ff1644ac6b7e1270d588a0614_90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:41:50,826 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/0ddbae73e35141c39ae9f8e6372a3d3f, store: [table=TestAcidGuarantees family=A region=90c0d1c8af782719d9f8d037e72c2a12] 2024-11-07T12:41:50,827 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/0ddbae73e35141c39ae9f8e6372a3d3f is 175, key is test_row_0/A:col10/1730983309586/Put/seqid=0 2024-11-07T12:41:50,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742000_1176 (size=39799) 2024-11-07T12:41:50,893 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=258, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/0ddbae73e35141c39ae9f8e6372a3d3f 2024-11-07T12:41:50,903 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/a1b2c69f572941ce81b8c9f43138631f is 50, key is test_row_0/B:col10/1730983309586/Put/seqid=0 2024-11-07T12:41:50,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:50,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43000 deadline: 1730983370902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:50,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:50,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43062 deadline: 1730983370903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:50,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:50,909 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:50,909 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:50,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:50,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:50,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:50,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:50,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43038 deadline: 1730983370912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:50,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:50,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:50,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:50,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742001_1177 (size=12201) 2024-11-07T12:41:50,919 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/a1b2c69f572941ce81b8c9f43138631f 2024-11-07T12:41:50,930 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/ef09e7c6663341d997fcb08c20339fd7 is 50, key is test_row_0/C:col10/1730983309586/Put/seqid=0 2024-11-07T12:41:50,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742002_1178 (size=12201) 2024-11-07T12:41:51,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:51,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43000 deadline: 1730983371112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:51,115 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:51,115 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:51,115 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:51,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:51,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43062 deadline: 1730983371122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:51,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:51,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43038 deadline: 1730983371122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:51,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:51,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:51,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:51,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:51,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:51,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:51,343 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/ef09e7c6663341d997fcb08c20339fd7 2024-11-07T12:41:51,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/0ddbae73e35141c39ae9f8e6372a3d3f as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/0ddbae73e35141c39ae9f8e6372a3d3f 2024-11-07T12:41:51,357 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/0ddbae73e35141c39ae9f8e6372a3d3f, entries=200, sequenceid=258, filesize=38.9 K 2024-11-07T12:41:51,359 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/a1b2c69f572941ce81b8c9f43138631f as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/a1b2c69f572941ce81b8c9f43138631f 2024-11-07T12:41:51,364 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/a1b2c69f572941ce81b8c9f43138631f, entries=150, sequenceid=258, filesize=11.9 K 2024-11-07T12:41:51,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/ef09e7c6663341d997fcb08c20339fd7 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/ef09e7c6663341d997fcb08c20339fd7 2024-11-07T12:41:51,371 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/ef09e7c6663341d997fcb08c20339fd7, entries=150, sequenceid=258, filesize=11.9 K 2024-11-07T12:41:51,372 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 90c0d1c8af782719d9f8d037e72c2a12 in 619ms, sequenceid=258, compaction requested=true 2024-11-07T12:41:51,372 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 90c0d1c8af782719d9f8d037e72c2a12: 2024-11-07T12:41:51,372 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:41:51,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 90c0d1c8af782719d9f8d037e72c2a12:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:41:51,374 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102453 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:41:51,374 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): 90c0d1c8af782719d9f8d037e72c2a12/A is initiating minor compaction (all files) 2024-11-07T12:41:51,374 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 90c0d1c8af782719d9f8d037e72c2a12/A in TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:51,374 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/7757eeeeef414dd5a23bcc6569148a3c, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/a0ee187b1da8408d9d73c194d72b4e01, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/0ddbae73e35141c39ae9f8e6372a3d3f] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp, totalSize=100.1 K 2024-11-07T12:41:51,374 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:51,374 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. files: [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/7757eeeeef414dd5a23bcc6569148a3c, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/a0ee187b1da8408d9d73c194d72b4e01, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/0ddbae73e35141c39ae9f8e6372a3d3f] 2024-11-07T12:41:51,375 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7757eeeeef414dd5a23bcc6569148a3c, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1730983308573 2024-11-07T12:41:51,375 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0ee187b1da8408d9d73c194d72b4e01, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1730983309229 2024-11-07T12:41:51,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:41:51,379 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:41:51,379 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ddbae73e35141c39ae9f8e6372a3d3f, keycount=200, bloomtype=ROW, size=38.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1730983309582 2024-11-07T12:41:51,381 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36947 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:41:51,381 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 90c0d1c8af782719d9f8d037e72c2a12/B is initiating minor compaction (all files) 2024-11-07T12:41:51,381 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 90c0d1c8af782719d9f8d037e72c2a12/B in TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:51,381 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/5684dac7279347ce9a65a3b1f952814d, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/e0f21398f98f49a58406a171ebd94b93, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/a1b2c69f572941ce81b8c9f43138631f] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp, totalSize=36.1 K 2024-11-07T12:41:51,382 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 5684dac7279347ce9a65a3b1f952814d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1730983308573 2024-11-07T12:41:51,383 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting e0f21398f98f49a58406a171ebd94b93, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1730983309229 2024-11-07T12:41:51,384 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting a1b2c69f572941ce81b8c9f43138631f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1730983309586 2024-11-07T12:41:51,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 90c0d1c8af782719d9f8d037e72c2a12:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:41:51,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:41:51,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 90c0d1c8af782719d9f8d037e72c2a12:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:41:51,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:41:51,394 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=90c0d1c8af782719d9f8d037e72c2a12] 2024-11-07T12:41:51,399 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411072608e6d9919b4d2d960ce9fcfac36eb1_90c0d1c8af782719d9f8d037e72c2a12 store=[table=TestAcidGuarantees family=A region=90c0d1c8af782719d9f8d037e72c2a12] 2024-11-07T12:41:51,401 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 90c0d1c8af782719d9f8d037e72c2a12#B#compaction#148 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:41:51,401 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411072608e6d9919b4d2d960ce9fcfac36eb1_90c0d1c8af782719d9f8d037e72c2a12, store=[table=TestAcidGuarantees family=A region=90c0d1c8af782719d9f8d037e72c2a12] 2024-11-07T12:41:51,401 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411072608e6d9919b4d2d960ce9fcfac36eb1_90c0d1c8af782719d9f8d037e72c2a12 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=90c0d1c8af782719d9f8d037e72c2a12] 2024-11-07T12:41:51,401 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/c7f6ca93f96544bd91bbc0a4c1c18f56 is 50, key is test_row_0/B:col10/1730983309586/Put/seqid=0 2024-11-07T12:41:51,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:41:51,429 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 90c0d1c8af782719d9f8d037e72c2a12 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-07T12:41:51,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 90c0d1c8af782719d9f8d037e72c2a12, store=A 2024-11-07T12:41:51,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:51,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 90c0d1c8af782719d9f8d037e72c2a12, store=B 2024-11-07T12:41:51,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:51,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 90c0d1c8af782719d9f8d037e72c2a12, store=C 2024-11-07T12:41:51,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:51,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742004_1180 (size=12747) 2024-11-07T12:41:51,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742003_1179 (size=4469) 2024-11-07T12:41:51,452 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107879f6a3833264cb08c1cf32d6f84f6ba_90c0d1c8af782719d9f8d037e72c2a12 is 50, key is test_row_0/A:col10/1730983310784/Put/seqid=0 2024-11-07T12:41:51,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742005_1181 (size=14994) 2024-11-07T12:41:51,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:51,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43038 deadline: 1730983371473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:51,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:51,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43062 deadline: 1730983371474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:51,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:51,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:51,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:51,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:51,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43000 deadline: 1730983371475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:51,483 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:51,483 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:51,483 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:51,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:51,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:51,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:51,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-07T12:41:51,532 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:41:51,533 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:41:51,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-07T12:41:51,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-07T12:41:51,535 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:41:51,536 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:41:51,536 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:41:51,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:51,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43038 deadline: 1730983371591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:51,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:51,593 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:51,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43000 deadline: 1730983371591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:51,593 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:51,593 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:51,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:51,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43062 deadline: 1730983371592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:51,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:51,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:51,594 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:51,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:51,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:51,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:51,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-07T12:41:51,690 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-07T12:41:51,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:51,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. as already flushing 2024-11-07T12:41:51,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:51,691 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:51,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:51,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:51,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:51,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43062 deadline: 1730983371802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:51,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:51,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:51,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:51,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43000 deadline: 1730983371802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:51,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:51,809 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:51,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:51,809 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:51,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43038 deadline: 1730983371802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:51,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:51,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:51,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:51,810 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:51,835 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 90c0d1c8af782719d9f8d037e72c2a12#A#compaction#147 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:41:51,836 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/2d9225edf52b4408a60fdf06c3aaef71 is 175, key is test_row_0/A:col10/1730983309586/Put/seqid=0 2024-11-07T12:41:51,839 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/c7f6ca93f96544bd91bbc0a4c1c18f56 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/c7f6ca93f96544bd91bbc0a4c1c18f56 2024-11-07T12:41:51,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742006_1182 (size=31701) 2024-11-07T12:41:51,844 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-07T12:41:51,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:51,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. as already flushing 2024-11-07T12:41:51,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:51,844 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:51,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:51,847 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 90c0d1c8af782719d9f8d037e72c2a12/B of 90c0d1c8af782719d9f8d037e72c2a12 into c7f6ca93f96544bd91bbc0a4c1c18f56(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:41:51,847 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 90c0d1c8af782719d9f8d037e72c2a12: 2024-11-07T12:41:51,847 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., storeName=90c0d1c8af782719d9f8d037e72c2a12/B, priority=13, startTime=1730983311379; duration=0sec 2024-11-07T12:41:51,848 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:41:51,848 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 90c0d1c8af782719d9f8d037e72c2a12:B 2024-11-07T12:41:51,848 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:41:51,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:51,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-07T12:41:51,855 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36947 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:41:51,855 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 90c0d1c8af782719d9f8d037e72c2a12/C is initiating minor compaction (all files) 2024-11-07T12:41:51,855 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 90c0d1c8af782719d9f8d037e72c2a12/C in TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:51,856 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/fe26956dc6e14c88b9c9d52c23372a57, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/76f0c164606340238fccfc5068df4d24, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/ef09e7c6663341d997fcb08c20339fd7] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp, totalSize=36.1 K 2024-11-07T12:41:51,857 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting fe26956dc6e14c88b9c9d52c23372a57, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1730983308573 2024-11-07T12:41:51,861 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 76f0c164606340238fccfc5068df4d24, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1730983309229 2024-11-07T12:41:51,861 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting ef09e7c6663341d997fcb08c20339fd7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1730983309586 2024-11-07T12:41:51,868 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/2d9225edf52b4408a60fdf06c3aaef71 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/2d9225edf52b4408a60fdf06c3aaef71 2024-11-07T12:41:51,905 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:51,908 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 90c0d1c8af782719d9f8d037e72c2a12/A of 90c0d1c8af782719d9f8d037e72c2a12 into 2d9225edf52b4408a60fdf06c3aaef71(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:41:51,908 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 90c0d1c8af782719d9f8d037e72c2a12: 2024-11-07T12:41:51,908 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., storeName=90c0d1c8af782719d9f8d037e72c2a12/A, priority=13, startTime=1730983311372; duration=0sec 2024-11-07T12:41:51,908 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:41:51,909 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 90c0d1c8af782719d9f8d037e72c2a12:A 2024-11-07T12:41:51,911 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 90c0d1c8af782719d9f8d037e72c2a12#C#compaction#150 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:41:51,912 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/adc90fca257d4c0a961c844b2eb1ac58 is 50, key is test_row_0/C:col10/1730983309586/Put/seqid=0 2024-11-07T12:41:51,915 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107879f6a3833264cb08c1cf32d6f84f6ba_90c0d1c8af782719d9f8d037e72c2a12 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107879f6a3833264cb08c1cf32d6f84f6ba_90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:41:51,917 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/ac0349aca1bf4e05a1b5886b70fb6cca, store: [table=TestAcidGuarantees family=A region=90c0d1c8af782719d9f8d037e72c2a12] 2024-11-07T12:41:51,917 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/ac0349aca1bf4e05a1b5886b70fb6cca is 175, key is test_row_0/A:col10/1730983310784/Put/seqid=0 2024-11-07T12:41:51,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742007_1183 (size=12747) 2024-11-07T12:41:51,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742008_1184 (size=39949) 2024-11-07T12:41:52,006 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-07T12:41:52,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:52,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. as already flushing 2024-11-07T12:41:52,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:52,006 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:52,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:52,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:52,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:52,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43062 deadline: 1730983372121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:52,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:52,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:52,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:52,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:52,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43038 deadline: 1730983372121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:52,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:52,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:52,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43000 deadline: 1730983372122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:52,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:52,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:52,124 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:52,124 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:52,124 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:52,160 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-07T12:41:52,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:52,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. as already flushing 2024-11-07T12:41:52,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:52,161 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:52,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:52,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:52,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-07T12:41:52,314 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-07T12:41:52,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:52,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. as already flushing 2024-11-07T12:41:52,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:52,314 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:52,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:52,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:52,348 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/adc90fca257d4c0a961c844b2eb1ac58 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/adc90fca257d4c0a961c844b2eb1ac58 2024-11-07T12:41:52,361 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 90c0d1c8af782719d9f8d037e72c2a12/C of 90c0d1c8af782719d9f8d037e72c2a12 into adc90fca257d4c0a961c844b2eb1ac58(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:41:52,361 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 90c0d1c8af782719d9f8d037e72c2a12: 2024-11-07T12:41:52,361 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., storeName=90c0d1c8af782719d9f8d037e72c2a12/C, priority=13, startTime=1730983311384; duration=0sec 2024-11-07T12:41:52,361 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:41:52,361 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 90c0d1c8af782719d9f8d037e72c2a12:C 2024-11-07T12:41:52,399 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=278, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/ac0349aca1bf4e05a1b5886b70fb6cca 2024-11-07T12:41:52,415 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/a35776c4f094458d81f13ad96df000f9 is 50, key is test_row_0/B:col10/1730983310784/Put/seqid=0 2024-11-07T12:41:52,469 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-07T12:41:52,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:52,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. as already flushing 2024-11-07T12:41:52,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:52,469 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:52,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:52,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742009_1185 (size=12301) 2024-11-07T12:41:52,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:52,625 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-07T12:41:52,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:52,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. as already flushing 2024-11-07T12:41:52,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:52,626 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:52,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:52,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:52,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:52,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43062 deadline: 1730983372632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:52,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:52,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43038 deadline: 1730983372633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:52,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:52,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:52,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:52,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:52,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43000 deadline: 1730983372632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:52,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:52,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:52,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:52,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:52,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:52,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:52,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-07T12:41:52,780 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-07T12:41:52,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:52,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. as already flushing 2024-11-07T12:41:52,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:52,780 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:52,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:52,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:52,860 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/a35776c4f094458d81f13ad96df000f9 2024-11-07T12:41:52,876 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/ee5d451d571b42f984324af0d399c642 is 50, key is test_row_0/C:col10/1730983310784/Put/seqid=0 2024-11-07T12:41:52,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742010_1186 (size=12301) 2024-11-07T12:41:52,883 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/ee5d451d571b42f984324af0d399c642 2024-11-07T12:41:52,888 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/ac0349aca1bf4e05a1b5886b70fb6cca as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/ac0349aca1bf4e05a1b5886b70fb6cca 2024-11-07T12:41:52,893 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/ac0349aca1bf4e05a1b5886b70fb6cca, entries=200, sequenceid=278, filesize=39.0 K 2024-11-07T12:41:52,895 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/a35776c4f094458d81f13ad96df000f9 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/a35776c4f094458d81f13ad96df000f9 2024-11-07T12:41:52,900 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/a35776c4f094458d81f13ad96df000f9, entries=150, sequenceid=278, filesize=12.0 K 2024-11-07T12:41:52,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/ee5d451d571b42f984324af0d399c642 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/ee5d451d571b42f984324af0d399c642 2024-11-07T12:41:52,919 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/ee5d451d571b42f984324af0d399c642, entries=150, sequenceid=278, filesize=12.0 K 2024-11-07T12:41:52,920 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 90c0d1c8af782719d9f8d037e72c2a12 in 1492ms, sequenceid=278, compaction requested=false 2024-11-07T12:41:52,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 90c0d1c8af782719d9f8d037e72c2a12: 2024-11-07T12:41:52,933 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-07T12:41:52,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:52,934 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 90c0d1c8af782719d9f8d037e72c2a12 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-07T12:41:52,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 90c0d1c8af782719d9f8d037e72c2a12, store=A 2024-11-07T12:41:52,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:52,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 90c0d1c8af782719d9f8d037e72c2a12, store=B 2024-11-07T12:41:52,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:52,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 90c0d1c8af782719d9f8d037e72c2a12, store=C 2024-11-07T12:41:52,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:52,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107bf81e9544c7d431bbb7ce9415d715358_90c0d1c8af782719d9f8d037e72c2a12 is 50, key is test_row_0/A:col10/1730983311473/Put/seqid=0 2024-11-07T12:41:52,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742011_1187 (size=12454) 2024-11-07T12:41:53,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:53,366 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107bf81e9544c7d431bbb7ce9415d715358_90c0d1c8af782719d9f8d037e72c2a12 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107bf81e9544c7d431bbb7ce9415d715358_90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:41:53,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/33c51829bec8419ba5996dde87d66d22, store: [table=TestAcidGuarantees family=A region=90c0d1c8af782719d9f8d037e72c2a12] 2024-11-07T12:41:53,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/33c51829bec8419ba5996dde87d66d22 is 175, key is test_row_0/A:col10/1730983311473/Put/seqid=0 2024-11-07T12:41:53,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742012_1188 (size=31255) 2024-11-07T12:41:53,381 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=294, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/33c51829bec8419ba5996dde87d66d22 2024-11-07T12:41:53,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/bbe5c39094634aa3bca8cedef889e925 is 50, key is test_row_0/B:col10/1730983311473/Put/seqid=0 2024-11-07T12:41:53,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742013_1189 (size=12301) 2024-11-07T12:41:53,443 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/bbe5c39094634aa3bca8cedef889e925 2024-11-07T12:41:53,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/c740cfd43765441f86fd60e108b4043c is 50, key is test_row_0/C:col10/1730983311473/Put/seqid=0 2024-11-07T12:41:53,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742014_1190 (size=12301) 2024-11-07T12:41:53,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:41:53,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. as already flushing 2024-11-07T12:41:53,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:53,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43062 deadline: 1730983373671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:53,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:53,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43000 deadline: 1730983373678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:53,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:53,680 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:53,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:53,681 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:53,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:53,681 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:53,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:53,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43038 deadline: 1730983373679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:53,685 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:53,686 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:53,686 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:53,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-07T12:41:53,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:53,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43062 deadline: 1730983373791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:53,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:53,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:53,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43000 deadline: 1730983373791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:53,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:53,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:53,797 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:53,797 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:53,797 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:53,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:53,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43038 deadline: 1730983373796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:53,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:53,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:53,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:53,866 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/c740cfd43765441f86fd60e108b4043c 2024-11-07T12:41:53,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/33c51829bec8419ba5996dde87d66d22 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/33c51829bec8419ba5996dde87d66d22 2024-11-07T12:41:53,891 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/33c51829bec8419ba5996dde87d66d22, entries=150, sequenceid=294, filesize=30.5 K 2024-11-07T12:41:53,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/bbe5c39094634aa3bca8cedef889e925 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/bbe5c39094634aa3bca8cedef889e925 2024-11-07T12:41:53,899 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/bbe5c39094634aa3bca8cedef889e925, entries=150, sequenceid=294, filesize=12.0 K 2024-11-07T12:41:53,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/c740cfd43765441f86fd60e108b4043c as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/c740cfd43765441f86fd60e108b4043c 2024-11-07T12:41:53,909 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/c740cfd43765441f86fd60e108b4043c, entries=150, sequenceid=294, filesize=12.0 K 2024-11-07T12:41:53,911 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 90c0d1c8af782719d9f8d037e72c2a12 in 978ms, sequenceid=294, compaction requested=true 2024-11-07T12:41:53,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 90c0d1c8af782719d9f8d037e72c2a12: 2024-11-07T12:41:53,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:53,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-07T12:41:53,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-07T12:41:53,916 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-07T12:41:53,916 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3770 sec 2024-11-07T12:41:53,918 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 2.3840 sec 2024-11-07T12:41:54,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:41:54,005 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 90c0d1c8af782719d9f8d037e72c2a12 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-07T12:41:54,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 90c0d1c8af782719d9f8d037e72c2a12, store=A 2024-11-07T12:41:54,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:41:54,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 90c0d1c8af782719d9f8d037e72c2a12, store=B 2024-11-07T12:41:54,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:41:54,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 90c0d1c8af782719d9f8d037e72c2a12, store=C 2024-11-07T12:41:54,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:41:54,018 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411070c9e13ad68724ad8a18c6f4b7f2e9078_90c0d1c8af782719d9f8d037e72c2a12 is 50, key is test_row_0/A:col10/1730983314004/Put/seqid=0 2024-11-07T12:41:54,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742015_1191 (size=14994) 2024-11-07T12:41:54,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:54,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43000 deadline: 1730983374069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:54,071 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:54,071 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:54,071 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:54,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:54,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43038 deadline: 1730983374070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:54,074 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:54,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:54,074 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:54,074 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:54,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43062 deadline: 1730983374071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:54,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:54,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:54,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:54,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:54,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43038 deadline: 1730983374182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:54,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:54,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:54,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:54,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:54,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43000 deadline: 1730983374183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:54,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:54,185 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:54,185 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:54,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43062 deadline: 1730983374183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:54,185 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:54,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:54,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:54,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:54,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:54,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43028 deadline: 1730983374193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:54,194 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to db9ad1cb6cf9,34293,1730983260217 for 'test_row_1' in 90c0d1c8af782719d9f8d037e72c2a12 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8241 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:41:54,195 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:54,195 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:54,195 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:54,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:54,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42994 deadline: 1730983374222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:54,230 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to db9ad1cb6cf9,34293,1730983260217 for 'test_row_0' in 90c0d1c8af782719d9f8d037e72c2a12 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8273 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:41:54,231 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:54,231 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:54,231 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:54,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:54,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43038 deadline: 1730983374391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:54,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:54,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:54,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:54,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:54,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43062 deadline: 1730983374392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:54,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:54,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43000 deadline: 1730983374392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:54,397 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:54,397 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:54,397 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:54,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:54,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:54,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:54,437 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:54,443 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411070c9e13ad68724ad8a18c6f4b7f2e9078_90c0d1c8af782719d9f8d037e72c2a12 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411070c9e13ad68724ad8a18c6f4b7f2e9078_90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:41:54,444 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/74723be07b174b008693cfd06571d16e, store: [table=TestAcidGuarantees family=A region=90c0d1c8af782719d9f8d037e72c2a12] 2024-11-07T12:41:54,445 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/74723be07b174b008693cfd06571d16e is 175, key is test_row_0/A:col10/1730983314004/Put/seqid=0 2024-11-07T12:41:54,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742016_1192 (size=39949) 2024-11-07T12:41:54,464 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=315, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/74723be07b174b008693cfd06571d16e 2024-11-07T12:41:54,477 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/0ab7a877c9174fa795ba8cbaed854bdc is 50, key is test_row_0/B:col10/1730983314004/Put/seqid=0 2024-11-07T12:41:54,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742017_1193 (size=12301) 2024-11-07T12:41:54,494 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/0ab7a877c9174fa795ba8cbaed854bdc 2024-11-07T12:41:54,532 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/5137832d790a494aa2c6df7b411f8380 is 50, key is test_row_0/C:col10/1730983314004/Put/seqid=0 2024-11-07T12:41:54,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742018_1194 (size=12301) 2024-11-07T12:41:54,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:54,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43000 deadline: 1730983374702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:54,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:54,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:54,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:54,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43038 deadline: 1730983374702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:54,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:54,703 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:54,703 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:54,703 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:54,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:54,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43062 deadline: 1730983374712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:54,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:54,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:54,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:54,964 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/5137832d790a494aa2c6df7b411f8380 2024-11-07T12:41:54,973 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/74723be07b174b008693cfd06571d16e as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/74723be07b174b008693cfd06571d16e 2024-11-07T12:41:54,978 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/74723be07b174b008693cfd06571d16e, entries=200, sequenceid=315, filesize=39.0 K 2024-11-07T12:41:54,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/0ab7a877c9174fa795ba8cbaed854bdc as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/0ab7a877c9174fa795ba8cbaed854bdc 2024-11-07T12:41:54,989 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/0ab7a877c9174fa795ba8cbaed854bdc, entries=150, sequenceid=315, filesize=12.0 K 2024-11-07T12:41:54,990 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/5137832d790a494aa2c6df7b411f8380 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/5137832d790a494aa2c6df7b411f8380 2024-11-07T12:41:54,997 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/5137832d790a494aa2c6df7b411f8380, entries=150, sequenceid=315, filesize=12.0 K 2024-11-07T12:41:54,998 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=67.09 KB/68700 for 90c0d1c8af782719d9f8d037e72c2a12 in 995ms, sequenceid=315, compaction requested=true 2024-11-07T12:41:54,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 90c0d1c8af782719d9f8d037e72c2a12: 2024-11-07T12:41:54,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 90c0d1c8af782719d9f8d037e72c2a12:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:41:54,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:41:54,998 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:41:54,998 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:41:54,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 90c0d1c8af782719d9f8d037e72c2a12:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:41:54,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:41:54,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 90c0d1c8af782719d9f8d037e72c2a12:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:41:54,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:41:55,001 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49650 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:41:55,001 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 90c0d1c8af782719d9f8d037e72c2a12/B is initiating minor compaction (all files) 2024-11-07T12:41:55,001 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 90c0d1c8af782719d9f8d037e72c2a12/B in TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:55,001 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/c7f6ca93f96544bd91bbc0a4c1c18f56, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/a35776c4f094458d81f13ad96df000f9, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/bbe5c39094634aa3bca8cedef889e925, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/0ab7a877c9174fa795ba8cbaed854bdc] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp, totalSize=48.5 K 2024-11-07T12:41:55,001 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142854 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:41:55,002 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): 90c0d1c8af782719d9f8d037e72c2a12/A is initiating minor compaction (all files) 2024-11-07T12:41:55,002 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 90c0d1c8af782719d9f8d037e72c2a12/A in TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:55,002 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/2d9225edf52b4408a60fdf06c3aaef71, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/ac0349aca1bf4e05a1b5886b70fb6cca, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/33c51829bec8419ba5996dde87d66d22, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/74723be07b174b008693cfd06571d16e] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp, totalSize=139.5 K 2024-11-07T12:41:55,002 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:55,002 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. files: [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/2d9225edf52b4408a60fdf06c3aaef71, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/ac0349aca1bf4e05a1b5886b70fb6cca, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/33c51829bec8419ba5996dde87d66d22, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/74723be07b174b008693cfd06571d16e] 2024-11-07T12:41:55,002 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting c7f6ca93f96544bd91bbc0a4c1c18f56, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1730983309586 2024-11-07T12:41:55,003 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d9225edf52b4408a60fdf06c3aaef71, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1730983309586 2024-11-07T12:41:55,003 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting a35776c4f094458d81f13ad96df000f9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1730983310784 2024-11-07T12:41:55,004 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac0349aca1bf4e05a1b5886b70fb6cca, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1730983310784 2024-11-07T12:41:55,004 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting bbe5c39094634aa3bca8cedef889e925, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1730983311471 2024-11-07T12:41:55,005 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33c51829bec8419ba5996dde87d66d22, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1730983311471 2024-11-07T12:41:55,005 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ab7a877c9174fa795ba8cbaed854bdc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1730983313668 2024-11-07T12:41:55,007 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74723be07b174b008693cfd06571d16e, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1730983313668 2024-11-07T12:41:55,031 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 90c0d1c8af782719d9f8d037e72c2a12#B#compaction#159 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:41:55,032 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/fd3db2b9a1a24a1bb39f0b43dfb029cc is 50, key is test_row_0/B:col10/1730983314004/Put/seqid=0 2024-11-07T12:41:55,034 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=90c0d1c8af782719d9f8d037e72c2a12] 2024-11-07T12:41:55,038 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241107177c7731959c4681a30e9fa0c37a5265_90c0d1c8af782719d9f8d037e72c2a12 store=[table=TestAcidGuarantees family=A region=90c0d1c8af782719d9f8d037e72c2a12] 2024-11-07T12:41:55,040 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241107177c7731959c4681a30e9fa0c37a5265_90c0d1c8af782719d9f8d037e72c2a12, store=[table=TestAcidGuarantees family=A region=90c0d1c8af782719d9f8d037e72c2a12] 2024-11-07T12:41:55,041 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107177c7731959c4681a30e9fa0c37a5265_90c0d1c8af782719d9f8d037e72c2a12 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=90c0d1c8af782719d9f8d037e72c2a12] 2024-11-07T12:41:55,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742019_1195 (size=12983) 2024-11-07T12:41:55,048 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/fd3db2b9a1a24a1bb39f0b43dfb029cc as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/fd3db2b9a1a24a1bb39f0b43dfb029cc 2024-11-07T12:41:55,054 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 90c0d1c8af782719d9f8d037e72c2a12/B of 90c0d1c8af782719d9f8d037e72c2a12 into fd3db2b9a1a24a1bb39f0b43dfb029cc(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:41:55,054 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 90c0d1c8af782719d9f8d037e72c2a12: 2024-11-07T12:41:55,054 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., storeName=90c0d1c8af782719d9f8d037e72c2a12/B, priority=12, startTime=1730983314998; duration=0sec 2024-11-07T12:41:55,054 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:41:55,054 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 90c0d1c8af782719d9f8d037e72c2a12:B 2024-11-07T12:41:55,054 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:41:55,055 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49650 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:41:55,056 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 90c0d1c8af782719d9f8d037e72c2a12/C is initiating minor compaction (all files) 2024-11-07T12:41:55,056 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 90c0d1c8af782719d9f8d037e72c2a12/C in TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:55,056 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/adc90fca257d4c0a961c844b2eb1ac58, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/ee5d451d571b42f984324af0d399c642, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/c740cfd43765441f86fd60e108b4043c, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/5137832d790a494aa2c6df7b411f8380] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp, totalSize=48.5 K 2024-11-07T12:41:55,056 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting adc90fca257d4c0a961c844b2eb1ac58, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1730983309586 2024-11-07T12:41:55,057 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting ee5d451d571b42f984324af0d399c642, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1730983310784 2024-11-07T12:41:55,057 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting c740cfd43765441f86fd60e108b4043c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1730983311471 2024-11-07T12:41:55,058 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 5137832d790a494aa2c6df7b411f8380, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1730983313668 2024-11-07T12:41:55,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742020_1196 (size=4469) 2024-11-07T12:41:55,070 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 90c0d1c8af782719d9f8d037e72c2a12#C#compaction#161 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:41:55,071 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/86f06de6234249afb55147aa826ae715 is 50, key is test_row_0/C:col10/1730983314004/Put/seqid=0 2024-11-07T12:41:55,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742021_1197 (size=12983) 2024-11-07T12:41:55,111 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/86f06de6234249afb55147aa826ae715 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/86f06de6234249afb55147aa826ae715 2024-11-07T12:41:55,119 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 90c0d1c8af782719d9f8d037e72c2a12/C of 90c0d1c8af782719d9f8d037e72c2a12 into 86f06de6234249afb55147aa826ae715(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:41:55,119 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 90c0d1c8af782719d9f8d037e72c2a12: 2024-11-07T12:41:55,119 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., storeName=90c0d1c8af782719d9f8d037e72c2a12/C, priority=12, startTime=1730983314999; duration=0sec 2024-11-07T12:41:55,119 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:41:55,119 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 90c0d1c8af782719d9f8d037e72c2a12:C 2024-11-07T12:41:55,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:41:55,214 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 90c0d1c8af782719d9f8d037e72c2a12 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-07T12:41:55,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 90c0d1c8af782719d9f8d037e72c2a12, store=A 2024-11-07T12:41:55,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:55,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 90c0d1c8af782719d9f8d037e72c2a12, store=B 2024-11-07T12:41:55,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:55,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 90c0d1c8af782719d9f8d037e72c2a12, store=C 2024-11-07T12:41:55,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:55,236 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110778f1add6eb834223a6ccc2b1b0d918fd_90c0d1c8af782719d9f8d037e72c2a12 is 50, key is test_row_0/A:col10/1730983315214/Put/seqid=0 2024-11-07T12:41:55,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742022_1198 (size=14994) 2024-11-07T12:41:55,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:55,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43062 deadline: 1730983375261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:55,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:55,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:55,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:55,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:55,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43038 deadline: 1730983375265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:55,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:55,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:55,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:55,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43000 deadline: 1730983375267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:55,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:55,268 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:55,268 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:55,268 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:55,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:55,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43062 deadline: 1730983375373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:55,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:55,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43000 deadline: 1730983375373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:55,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:55,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43038 deadline: 1730983375373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:55,382 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:55,382 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:55,382 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:55,383 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:55,383 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:55,383 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:55,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:55,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:55,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:55,466 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 90c0d1c8af782719d9f8d037e72c2a12#A#compaction#160 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:41:55,467 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/dd88931613b04986b4630968c2713385 is 175, key is test_row_0/A:col10/1730983314004/Put/seqid=0 2024-11-07T12:41:55,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742023_1199 (size=31937) 2024-11-07T12:41:55,506 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/dd88931613b04986b4630968c2713385 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/dd88931613b04986b4630968c2713385 2024-11-07T12:41:55,524 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 90c0d1c8af782719d9f8d037e72c2a12/A of 90c0d1c8af782719d9f8d037e72c2a12 into dd88931613b04986b4630968c2713385(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:41:55,524 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 90c0d1c8af782719d9f8d037e72c2a12: 2024-11-07T12:41:55,524 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., storeName=90c0d1c8af782719d9f8d037e72c2a12/A, priority=12, startTime=1730983314998; duration=0sec 2024-11-07T12:41:55,525 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:41:55,525 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 90c0d1c8af782719d9f8d037e72c2a12:A 2024-11-07T12:41:55,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:55,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43000 deadline: 1730983375593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:55,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:55,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43038 deadline: 1730983375594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:55,595 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:55,595 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:55,595 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:55,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:55,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:55,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:55,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:55,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43062 deadline: 1730983375595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:55,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:55,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:55,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:55,649 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:55,671 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110778f1add6eb834223a6ccc2b1b0d918fd_90c0d1c8af782719d9f8d037e72c2a12 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110778f1add6eb834223a6ccc2b1b0d918fd_90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:41:55,672 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/d100ce85e2a94aa1b4f7303f1c43016b, store: [table=TestAcidGuarantees family=A region=90c0d1c8af782719d9f8d037e72c2a12] 2024-11-07T12:41:55,673 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/d100ce85e2a94aa1b4f7303f1c43016b is 175, key is test_row_0/A:col10/1730983315214/Put/seqid=0 2024-11-07T12:41:55,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-07T12:41:55,712 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:41:55,713 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:41:55,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742024_1200 (size=39949) 2024-11-07T12:41:55,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-07T12:41:55,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-07T12:41:55,733 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:41:55,747 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:41:55,747 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:41:55,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-07T12:41:55,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:55,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43038 deadline: 1730983375902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:55,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:55,907 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:55,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43000 deadline: 1730983375902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:55,907 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:55,907 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:55,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:55,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43062 deadline: 1730983375902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:55,907 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:55,907 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:55,907 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:55,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:55,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:55,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:55,916 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-07T12:41:55,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:55,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. as already flushing 2024-11-07T12:41:55,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:55,916 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:55,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:55,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:56,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-07T12:41:56,071 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-07T12:41:56,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:56,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. as already flushing 2024-11-07T12:41:56,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:56,072 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:56,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:56,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:56,120 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=332, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/d100ce85e2a94aa1b4f7303f1c43016b 2024-11-07T12:41:56,134 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/6401b48fb2c842419dccf7474c63c948 is 50, key is test_row_0/B:col10/1730983315214/Put/seqid=0 2024-11-07T12:41:56,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742025_1201 (size=12301) 2024-11-07T12:41:56,225 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-07T12:41:56,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:56,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. as already flushing 2024-11-07T12:41:56,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:56,228 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:56,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:56,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:56,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-07T12:41:56,381 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-07T12:41:56,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:56,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. as already flushing 2024-11-07T12:41:56,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:56,381 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:56,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:56,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:56,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:56,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43038 deadline: 1730983376411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:56,413 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:56,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:56,413 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:56,413 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:56,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43062 deadline: 1730983376412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:56,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:56,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:56,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:56,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:56,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43000 deadline: 1730983376422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:56,424 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:56,424 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:56,425 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:56,535 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-07T12:41:56,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:56,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. as already flushing 2024-11-07T12:41:56,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:56,536 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:56,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:56,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:56,548 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/6401b48fb2c842419dccf7474c63c948 2024-11-07T12:41:56,558 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/b921047a98044531af923a9231c3f798 is 50, key is test_row_0/C:col10/1730983315214/Put/seqid=0 2024-11-07T12:41:56,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742026_1202 (size=12301) 2024-11-07T12:41:56,564 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/b921047a98044531af923a9231c3f798 2024-11-07T12:41:56,569 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/d100ce85e2a94aa1b4f7303f1c43016b as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/d100ce85e2a94aa1b4f7303f1c43016b 2024-11-07T12:41:56,577 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/d100ce85e2a94aa1b4f7303f1c43016b, entries=200, sequenceid=332, filesize=39.0 K 2024-11-07T12:41:56,578 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/6401b48fb2c842419dccf7474c63c948 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/6401b48fb2c842419dccf7474c63c948 2024-11-07T12:41:56,582 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/6401b48fb2c842419dccf7474c63c948, entries=150, sequenceid=332, filesize=12.0 K 2024-11-07T12:41:56,583 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/b921047a98044531af923a9231c3f798 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/b921047a98044531af923a9231c3f798 2024-11-07T12:41:56,588 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/b921047a98044531af923a9231c3f798, entries=150, sequenceid=332, filesize=12.0 K 2024-11-07T12:41:56,589 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 90c0d1c8af782719d9f8d037e72c2a12 in 1375ms, sequenceid=332, compaction requested=false 2024-11-07T12:41:56,590 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 90c0d1c8af782719d9f8d037e72c2a12: 2024-11-07T12:41:56,689 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-07T12:41:56,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:56,690 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 90c0d1c8af782719d9f8d037e72c2a12 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-07T12:41:56,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 90c0d1c8af782719d9f8d037e72c2a12, store=A 2024-11-07T12:41:56,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:56,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 90c0d1c8af782719d9f8d037e72c2a12, store=B 2024-11-07T12:41:56,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:56,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 90c0d1c8af782719d9f8d037e72c2a12, store=C 2024-11-07T12:41:56,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:56,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411074bcd381d40884f7fb44bf3bac7c7d4f5_90c0d1c8af782719d9f8d037e72c2a12 is 50, key is test_row_0/A:col10/1730983315248/Put/seqid=0 2024-11-07T12:41:56,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742027_1203 (size=12454) 2024-11-07T12:41:56,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,723 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411074bcd381d40884f7fb44bf3bac7c7d4f5_90c0d1c8af782719d9f8d037e72c2a12 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411074bcd381d40884f7fb44bf3bac7c7d4f5_90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:41:56,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/011f62f1ad644c35b58c0d47491725ac, store: [table=TestAcidGuarantees family=A region=90c0d1c8af782719d9f8d037e72c2a12] 2024-11-07T12:41:56,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/011f62f1ad644c35b58c0d47491725ac is 175, key is test_row_0/A:col10/1730983315248/Put/seqid=0 2024-11-07T12:41:56,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742028_1204 (size=31255) 2024-11-07T12:41:56,729 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=351, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/011f62f1ad644c35b58c0d47491725ac 2024-11-07T12:41:56,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/5536db3c5ee6483ba192deacacfcf700 is 50, key is test_row_0/B:col10/1730983315248/Put/seqid=0 2024-11-07T12:41:56,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742029_1205 (size=12301) 2024-11-07T12:41:56,773 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/5536db3c5ee6483ba192deacacfcf700 2024-11-07T12:41:56,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/1c04e825af2043ed972d46ce385a545c is 50, key is test_row_0/C:col10/1730983315248/Put/seqid=0 2024-11-07T12:41:56,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742030_1206 (size=12301) 2024-11-07T12:41:56,825 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/1c04e825af2043ed972d46ce385a545c 2024-11-07T12:41:56,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/011f62f1ad644c35b58c0d47491725ac as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/011f62f1ad644c35b58c0d47491725ac 2024-11-07T12:41:56,839 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/011f62f1ad644c35b58c0d47491725ac, entries=150, sequenceid=351, filesize=30.5 K 2024-11-07T12:41:56,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/5536db3c5ee6483ba192deacacfcf700 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/5536db3c5ee6483ba192deacacfcf700 2024-11-07T12:41:56,846 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/5536db3c5ee6483ba192deacacfcf700, entries=150, sequenceid=351, filesize=12.0 K 2024-11-07T12:41:56,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/1c04e825af2043ed972d46ce385a545c as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/1c04e825af2043ed972d46ce385a545c 2024-11-07T12:41:56,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,852 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/1c04e825af2043ed972d46ce385a545c, entries=150, sequenceid=351, filesize=12.0 K 2024-11-07T12:41:56,853 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=0 B/0 for 90c0d1c8af782719d9f8d037e72c2a12 in 164ms, sequenceid=351, compaction requested=true 2024-11-07T12:41:56,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 90c0d1c8af782719d9f8d037e72c2a12: 2024-11-07T12:41:56,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:56,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-07T12:41:56,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-07T12:41:56,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,858 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-07T12:41:56,858 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1080 sec 2024-11-07T12:41:56,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,861 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.1460 sec 2024-11-07T12:41:56,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-07T12:41:56,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,865 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:41:56,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,866 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:41:56,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-07T12:41:56,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-07T12:41:56,869 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:41:56,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,870 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:41:56,870 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:41:56,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-07T12:41:56,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:56,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,025 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-07T12:41:57,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:57,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 90c0d1c8af782719d9f8d037e72c2a12: 2024-11-07T12:41:57,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:57,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-07T12:41:57,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-07T12:41:57,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,030 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-07T12:41:57,030 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 157 msec 2024-11-07T12:41:57,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,039 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 165 msec 2024-11-07T12:41:57,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-07T12:41:57,182 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:41:57,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,182 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:41:57,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-07T12:41:57,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-07T12:41:57,185 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:41:57,185 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:41:57,186 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:41:57,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-07T12:41:57,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,339 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-07T12:41:57,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:57,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 90c0d1c8af782719d9f8d037e72c2a12: 2024-11-07T12:41:57,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:57,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-07T12:41:57,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-07T12:41:57,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,344 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-07T12:41:57,344 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 155 msec 2024-11-07T12:41:57,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,346 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 162 msec 2024-11-07T12:41:57,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-07T12:41:57,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,502 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:41:57,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,502 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:41:57,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-07T12:41:57,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,504 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:41:57,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-07T12:41:57,505 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:41:57,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,505 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:41:57,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,561 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 90c0d1c8af782719d9f8d037e72c2a12 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-07T12:41:57,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:41:57,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 90c0d1c8af782719d9f8d037e72c2a12, store=A 2024-11-07T12:41:57,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:57,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 90c0d1c8af782719d9f8d037e72c2a12, store=B 2024-11-07T12:41:57,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:57,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 90c0d1c8af782719d9f8d037e72c2a12, store=C 2024-11-07T12:41:57,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:57,588 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107ed27035721094552ad73611ff0ec234f_90c0d1c8af782719d9f8d037e72c2a12 is 50, key is test_row_0/A:col10/1730983317561/Put/seqid=0 2024-11-07T12:41:57,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-07T12:41:57,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742031_1207 (size=14994) 2024-11-07T12:41:57,631 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:57,636 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107ed27035721094552ad73611ff0ec234f_90c0d1c8af782719d9f8d037e72c2a12 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107ed27035721094552ad73611ff0ec234f_90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:41:57,637 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/87a2ffdf4a22414e84ef6cb022490537, store: [table=TestAcidGuarantees family=A region=90c0d1c8af782719d9f8d037e72c2a12] 2024-11-07T12:41:57,638 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/87a2ffdf4a22414e84ef6cb022490537 is 175, key is test_row_0/A:col10/1730983317561/Put/seqid=0 2024-11-07T12:41:57,644 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1730983317625, max timestamp=1730983317640Num uniques 859; 2024-11-07T12:41:57,645 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-07T12:41:57,650 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1730983317625, max timestamp=1730983317640Num uniques 859; 2024-11-07T12:41:57,650 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-07T12:41:57,647 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1730983317625, max timestamp=1730983317640Num uniques 859; 2024-11-07T12:41:57,651 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-07T12:41:57,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742032_1208 (size=39949) 2024-11-07T12:41:57,657 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=364, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/87a2ffdf4a22414e84ef6cb022490537 2024-11-07T12:41:57,659 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-07T12:41:57,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:57,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. as already flushing 2024-11-07T12:41:57,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:57,659 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:57,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:57,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:57,670 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/5bdd9ae4a196473db727833e65b5e0ae is 50, key is test_row_0/B:col10/1730983317561/Put/seqid=0 2024-11-07T12:41:57,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:57,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43038 deadline: 1730983377679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:57,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:57,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:57,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:57,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:57,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43000 deadline: 1730983377683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:57,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:57,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43062 deadline: 1730983377684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:57,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:57,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:57,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:57,695 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:57,695 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:57,695 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:57,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742033_1209 (size=12301) 2024-11-07T12:41:57,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:57,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43038 deadline: 1730983377791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:57,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:57,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:57,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:57,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:57,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43062 deadline: 1730983377802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:57,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:57,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:57,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:57,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:57,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43000 deadline: 1730983377802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:57,807 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:57,807 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:57,807 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:57,813 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-07T12:41:57,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:57,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. as already flushing 2024-11-07T12:41:57,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:57,813 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:57,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:57,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:57,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-07T12:41:57,837 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-07T12:41:57,967 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-07T12:41:57,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:57,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. as already flushing 2024-11-07T12:41:57,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:57,967 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:57,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:57,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:58,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:58,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43038 deadline: 1730983378002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:58,007 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:58,007 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:58,007 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:58,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:58,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43062 deadline: 1730983378012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:58,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:58,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:58,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:58,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:58,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43000 deadline: 1730983378012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:58,017 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:58,017 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:58,017 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:58,097 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/5bdd9ae4a196473db727833e65b5e0ae 2024-11-07T12:41:58,108 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/2ad36ff40b45481dac6e548f6b7f42fd is 50, key is test_row_0/C:col10/1730983317561/Put/seqid=0 2024-11-07T12:41:58,121 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-07T12:41:58,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:58,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. as already flushing 2024-11-07T12:41:58,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:58,121 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:58,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:58,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:41:58,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-07T12:41:58,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742034_1210 (size=12301) 2024-11-07T12:41:58,139 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/2ad36ff40b45481dac6e548f6b7f42fd 2024-11-07T12:41:58,145 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/87a2ffdf4a22414e84ef6cb022490537 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/87a2ffdf4a22414e84ef6cb022490537 2024-11-07T12:41:58,161 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/87a2ffdf4a22414e84ef6cb022490537, entries=200, sequenceid=364, filesize=39.0 K 2024-11-07T12:41:58,162 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/5bdd9ae4a196473db727833e65b5e0ae as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/5bdd9ae4a196473db727833e65b5e0ae 2024-11-07T12:41:58,168 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/5bdd9ae4a196473db727833e65b5e0ae, entries=150, sequenceid=364, filesize=12.0 K 2024-11-07T12:41:58,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/2ad36ff40b45481dac6e548f6b7f42fd as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/2ad36ff40b45481dac6e548f6b7f42fd 2024-11-07T12:41:58,174 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/2ad36ff40b45481dac6e548f6b7f42fd, entries=150, sequenceid=364, filesize=12.0 K 2024-11-07T12:41:58,175 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 90c0d1c8af782719d9f8d037e72c2a12 in 614ms, sequenceid=364, compaction requested=true 2024-11-07T12:41:58,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 90c0d1c8af782719d9f8d037e72c2a12: 2024-11-07T12:41:58,176 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:41:58,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 90c0d1c8af782719d9f8d037e72c2a12:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:41:58,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:41:58,176 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:41:58,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 90c0d1c8af782719d9f8d037e72c2a12:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:41:58,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:41:58,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 90c0d1c8af782719d9f8d037e72c2a12:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:41:58,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:41:58,177 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 143090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:41:58,177 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): 90c0d1c8af782719d9f8d037e72c2a12/A is initiating minor compaction (all files) 2024-11-07T12:41:58,178 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 90c0d1c8af782719d9f8d037e72c2a12/A in TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:58,178 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/dd88931613b04986b4630968c2713385, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/d100ce85e2a94aa1b4f7303f1c43016b, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/011f62f1ad644c35b58c0d47491725ac, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/87a2ffdf4a22414e84ef6cb022490537] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp, totalSize=139.7 K 2024-11-07T12:41:58,178 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:58,178 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. files: [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/dd88931613b04986b4630968c2713385, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/d100ce85e2a94aa1b4f7303f1c43016b, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/011f62f1ad644c35b58c0d47491725ac, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/87a2ffdf4a22414e84ef6cb022490537] 2024-11-07T12:41:58,179 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd88931613b04986b4630968c2713385, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1730983313668 2024-11-07T12:41:58,179 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:41:58,179 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 90c0d1c8af782719d9f8d037e72c2a12/B is initiating minor compaction (all files) 2024-11-07T12:41:58,179 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 90c0d1c8af782719d9f8d037e72c2a12/B in TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:58,179 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/fd3db2b9a1a24a1bb39f0b43dfb029cc, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/6401b48fb2c842419dccf7474c63c948, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/5536db3c5ee6483ba192deacacfcf700, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/5bdd9ae4a196473db727833e65b5e0ae] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp, totalSize=48.7 K 2024-11-07T12:41:58,180 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting d100ce85e2a94aa1b4f7303f1c43016b, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1730983314009 2024-11-07T12:41:58,181 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting fd3db2b9a1a24a1bb39f0b43dfb029cc, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1730983313668 2024-11-07T12:41:58,181 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 011f62f1ad644c35b58c0d47491725ac, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1730983315246 2024-11-07T12:41:58,181 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 6401b48fb2c842419dccf7474c63c948, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1730983314066 2024-11-07T12:41:58,181 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87a2ffdf4a22414e84ef6cb022490537, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1730983317490 2024-11-07T12:41:58,181 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 5536db3c5ee6483ba192deacacfcf700, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1730983315246 2024-11-07T12:41:58,182 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 5bdd9ae4a196473db727833e65b5e0ae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1730983317490 2024-11-07T12:41:58,197 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=90c0d1c8af782719d9f8d037e72c2a12] 2024-11-07T12:41:58,203 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 90c0d1c8af782719d9f8d037e72c2a12#B#compaction#172 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:41:58,204 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/16fa37c0370e4a52a7ffa9ed5907ac7e is 50, key is test_row_0/B:col10/1730983317561/Put/seqid=0 2024-11-07T12:41:58,206 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411077ee578396b40469891d3f6002f3d1344_90c0d1c8af782719d9f8d037e72c2a12 store=[table=TestAcidGuarantees family=A region=90c0d1c8af782719d9f8d037e72c2a12] 2024-11-07T12:41:58,208 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411077ee578396b40469891d3f6002f3d1344_90c0d1c8af782719d9f8d037e72c2a12, store=[table=TestAcidGuarantees family=A region=90c0d1c8af782719d9f8d037e72c2a12] 2024-11-07T12:41:58,209 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411077ee578396b40469891d3f6002f3d1344_90c0d1c8af782719d9f8d037e72c2a12 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=90c0d1c8af782719d9f8d037e72c2a12] 2024-11-07T12:41:58,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742036_1212 (size=4469) 2024-11-07T12:41:58,220 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 90c0d1c8af782719d9f8d037e72c2a12#A#compaction#171 average throughput is 1.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:41:58,221 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/1e8b21971162423bb6c1d58f6d402e12 is 175, key is test_row_0/A:col10/1730983317561/Put/seqid=0 2024-11-07T12:41:58,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742035_1211 (size=13119) 2024-11-07T12:41:58,244 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/16fa37c0370e4a52a7ffa9ed5907ac7e as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/16fa37c0370e4a52a7ffa9ed5907ac7e 2024-11-07T12:41:58,252 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 90c0d1c8af782719d9f8d037e72c2a12/B of 90c0d1c8af782719d9f8d037e72c2a12 into 16fa37c0370e4a52a7ffa9ed5907ac7e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:41:58,252 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 90c0d1c8af782719d9f8d037e72c2a12: 2024-11-07T12:41:58,252 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., storeName=90c0d1c8af782719d9f8d037e72c2a12/B, priority=12, startTime=1730983318176; duration=0sec 2024-11-07T12:41:58,253 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:41:58,253 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 90c0d1c8af782719d9f8d037e72c2a12:B 2024-11-07T12:41:58,253 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:41:58,254 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:41:58,254 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 90c0d1c8af782719d9f8d037e72c2a12/C is initiating minor compaction (all files) 2024-11-07T12:41:58,254 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 90c0d1c8af782719d9f8d037e72c2a12/C in TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:58,254 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/86f06de6234249afb55147aa826ae715, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/b921047a98044531af923a9231c3f798, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/1c04e825af2043ed972d46ce385a545c, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/2ad36ff40b45481dac6e548f6b7f42fd] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp, totalSize=48.7 K 2024-11-07T12:41:58,255 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 86f06de6234249afb55147aa826ae715, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1730983313668 2024-11-07T12:41:58,255 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting b921047a98044531af923a9231c3f798, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1730983314066 2024-11-07T12:41:58,256 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c04e825af2043ed972d46ce385a545c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1730983315246 2024-11-07T12:41:58,256 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ad36ff40b45481dac6e548f6b7f42fd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1730983317490 2024-11-07T12:41:58,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742037_1213 (size=32073) 2024-11-07T12:41:58,274 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-07T12:41:58,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:58,274 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 90c0d1c8af782719d9f8d037e72c2a12 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-11-07T12:41:58,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 90c0d1c8af782719d9f8d037e72c2a12, store=A 2024-11-07T12:41:58,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:41:58,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 90c0d1c8af782719d9f8d037e72c2a12, store=B 2024-11-07T12:41:58,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:41:58,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 90c0d1c8af782719d9f8d037e72c2a12, store=C 2024-11-07T12:41:58,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:41:58,280 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 90c0d1c8af782719d9f8d037e72c2a12#C#compaction#173 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:41:58,280 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/f702b6d4c1e04058bfb890407f6c1073 is 50, key is test_row_0/C:col10/1730983317561/Put/seqid=0 2024-11-07T12:41:58,285 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/1e8b21971162423bb6c1d58f6d402e12 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/1e8b21971162423bb6c1d58f6d402e12 2024-11-07T12:41:58,292 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 90c0d1c8af782719d9f8d037e72c2a12/A of 90c0d1c8af782719d9f8d037e72c2a12 into 1e8b21971162423bb6c1d58f6d402e12(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:41:58,292 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 90c0d1c8af782719d9f8d037e72c2a12: 2024-11-07T12:41:58,292 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., storeName=90c0d1c8af782719d9f8d037e72c2a12/A, priority=12, startTime=1730983318175; duration=0sec 2024-11-07T12:41:58,292 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:41:58,294 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 90c0d1c8af782719d9f8d037e72c2a12:A 2024-11-07T12:41:58,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107d44d9f700ec04d4db39c065fbc5d6969_90c0d1c8af782719d9f8d037e72c2a12 is 50, key is test_row_0/A:col10/1730983317683/Put/seqid=0 2024-11-07T12:41:58,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742038_1214 (size=13119) 2024-11-07T12:41:58,308 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/f702b6d4c1e04058bfb890407f6c1073 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/f702b6d4c1e04058bfb890407f6c1073 2024-11-07T12:41:58,313 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 90c0d1c8af782719d9f8d037e72c2a12/C of 90c0d1c8af782719d9f8d037e72c2a12 into f702b6d4c1e04058bfb890407f6c1073(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:41:58,313 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 90c0d1c8af782719d9f8d037e72c2a12: 2024-11-07T12:41:58,313 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., storeName=90c0d1c8af782719d9f8d037e72c2a12/C, priority=12, startTime=1730983318177; duration=0sec 2024-11-07T12:41:58,314 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:41:58,314 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 90c0d1c8af782719d9f8d037e72c2a12:C 2024-11-07T12:41:58,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:41:58,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. as already flushing 2024-11-07T12:41:58,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742039_1215 (size=12454) 2024-11-07T12:41:58,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:58,341 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107d44d9f700ec04d4db39c065fbc5d6969_90c0d1c8af782719d9f8d037e72c2a12 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107d44d9f700ec04d4db39c065fbc5d6969_90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:41:58,341 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:58,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43062 deadline: 1730983378337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:58,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:58,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:58,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:58,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/4fe985879e5d4292bdd82491d5ba6412, store: [table=TestAcidGuarantees family=A region=90c0d1c8af782719d9f8d037e72c2a12] 2024-11-07T12:41:58,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/4fe985879e5d4292bdd82491d5ba6412 is 175, key is test_row_0/A:col10/1730983317683/Put/seqid=0 2024-11-07T12:41:58,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:58,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43000 deadline: 1730983378342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:58,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:58,352 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:58,352 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:58,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:58,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43038 deadline: 1730983378342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:58,352 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:58,352 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:58,352 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:58,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742040_1216 (size=31255) 2024-11-07T12:41:58,359 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=399, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/4fe985879e5d4292bdd82491d5ba6412 2024-11-07T12:41:58,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/0e72c86d058a456ea665880421fcdb14 is 50, key is test_row_0/B:col10/1730983317683/Put/seqid=0 2024-11-07T12:41:58,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742041_1217 (size=12301) 2024-11-07T12:41:58,382 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/0e72c86d058a456ea665880421fcdb14 2024-11-07T12:41:58,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/0adf79712f2f4da08d9e75e8a8e06d4a is 50, key is test_row_0/C:col10/1730983317683/Put/seqid=0 2024-11-07T12:41:58,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742042_1218 (size=12301) 2024-11-07T12:41:58,404 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/0adf79712f2f4da08d9e75e8a8e06d4a 2024-11-07T12:41:58,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/4fe985879e5d4292bdd82491d5ba6412 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/4fe985879e5d4292bdd82491d5ba6412 2024-11-07T12:41:58,416 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/4fe985879e5d4292bdd82491d5ba6412, entries=150, sequenceid=399, filesize=30.5 K 2024-11-07T12:41:58,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/0e72c86d058a456ea665880421fcdb14 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/0e72c86d058a456ea665880421fcdb14 2024-11-07T12:41:58,421 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/0e72c86d058a456ea665880421fcdb14, entries=150, sequenceid=399, filesize=12.0 K 2024-11-07T12:41:58,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/0adf79712f2f4da08d9e75e8a8e06d4a as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/0adf79712f2f4da08d9e75e8a8e06d4a 2024-11-07T12:41:58,427 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/0adf79712f2f4da08d9e75e8a8e06d4a, entries=150, sequenceid=399, filesize=12.0 K 2024-11-07T12:41:58,428 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=53.67 KB/54960 for 90c0d1c8af782719d9f8d037e72c2a12 in 154ms, sequenceid=399, compaction requested=false 2024-11-07T12:41:58,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 90c0d1c8af782719d9f8d037e72c2a12: 2024-11-07T12:41:58,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:58,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-07T12:41:58,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-07T12:41:58,434 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-07T12:41:58,434 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 926 msec 2024-11-07T12:41:58,437 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 932 msec 2024-11-07T12:41:58,456 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 90c0d1c8af782719d9f8d037e72c2a12 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-07T12:41:58,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 90c0d1c8af782719d9f8d037e72c2a12, store=A 2024-11-07T12:41:58,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:58,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 90c0d1c8af782719d9f8d037e72c2a12, store=B 2024-11-07T12:41:58,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:58,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 90c0d1c8af782719d9f8d037e72c2a12, store=C 2024-11-07T12:41:58,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:58,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:41:58,478 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107f604959604024fdb85ef7e58f73aeeca_90c0d1c8af782719d9f8d037e72c2a12 is 50, key is test_row_0/A:col10/1730983318337/Put/seqid=0 2024-11-07T12:41:58,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742043_1219 (size=14994) 2024-11-07T12:41:58,526 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:58,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:58,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43000 deadline: 1730983378513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:58,526 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:58,526 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:58,526 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:58,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:58,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43038 deadline: 1730983378523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:58,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:58,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:58,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:58,530 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107f604959604024fdb85ef7e58f73aeeca_90c0d1c8af782719d9f8d037e72c2a12 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107f604959604024fdb85ef7e58f73aeeca_90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:41:58,532 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/b89ab59d63bd489e878a3ad332d74e07, store: [table=TestAcidGuarantees family=A region=90c0d1c8af782719d9f8d037e72c2a12] 2024-11-07T12:41:58,532 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/b89ab59d63bd489e878a3ad332d74e07 is 175, key is test_row_0/A:col10/1730983318337/Put/seqid=0 2024-11-07T12:41:58,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:58,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43062 deadline: 1730983378531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:58,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:58,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:58,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:58,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742044_1220 (size=39949) 2024-11-07T12:41:58,563 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=413, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/b89ab59d63bd489e878a3ad332d74e07 2024-11-07T12:41:58,574 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/5847bb74384a456495abbf3eb36c3603 is 50, key is test_row_0/B:col10/1730983318337/Put/seqid=0 2024-11-07T12:41:58,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742045_1221 (size=12301) 2024-11-07T12:41:58,618 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/5847bb74384a456495abbf3eb36c3603 2024-11-07T12:41:58,627 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/dcfb05bb7fb64180a00ea8b4781f075c is 50, key is test_row_0/C:col10/1730983318337/Put/seqid=0 2024-11-07T12:41:58,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:58,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43000 deadline: 1730983378632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:58,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:58,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43038 deadline: 1730983378632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:58,639 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:58,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:58,639 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:58,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:58,639 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:58,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:58,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-07T12:41:58,642 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:41:58,643 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:41:58,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-07T12:41:58,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-07T12:41:58,646 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:41:58,647 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:41:58,648 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:41:58,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742046_1222 (size=12301) 2024-11-07T12:41:58,651 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/dcfb05bb7fb64180a00ea8b4781f075c 2024-11-07T12:41:58,657 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/b89ab59d63bd489e878a3ad332d74e07 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/b89ab59d63bd489e878a3ad332d74e07 2024-11-07T12:41:58,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:58,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43062 deadline: 1730983378652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:58,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:58,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:58,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:58,662 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/b89ab59d63bd489e878a3ad332d74e07, entries=200, sequenceid=413, filesize=39.0 K 2024-11-07T12:41:58,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/5847bb74384a456495abbf3eb36c3603 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/5847bb74384a456495abbf3eb36c3603 2024-11-07T12:41:58,671 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/5847bb74384a456495abbf3eb36c3603, entries=150, sequenceid=413, filesize=12.0 K 2024-11-07T12:41:58,672 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/dcfb05bb7fb64180a00ea8b4781f075c as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/dcfb05bb7fb64180a00ea8b4781f075c 2024-11-07T12:41:58,690 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/dcfb05bb7fb64180a00ea8b4781f075c, entries=150, sequenceid=413, filesize=12.0 K 2024-11-07T12:41:58,691 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 90c0d1c8af782719d9f8d037e72c2a12 in 235ms, sequenceid=413, compaction requested=true 2024-11-07T12:41:58,691 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 90c0d1c8af782719d9f8d037e72c2a12: 2024-11-07T12:41:58,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 90c0d1c8af782719d9f8d037e72c2a12:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:41:58,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:41:58,691 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:41:58,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 90c0d1c8af782719d9f8d037e72c2a12:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:41:58,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:41:58,692 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:41:58,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 90c0d1c8af782719d9f8d037e72c2a12:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:41:58,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:41:58,693 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103277 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:41:58,693 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): 90c0d1c8af782719d9f8d037e72c2a12/A is initiating minor compaction (all files) 2024-11-07T12:41:58,693 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 90c0d1c8af782719d9f8d037e72c2a12/A in TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:58,693 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/1e8b21971162423bb6c1d58f6d402e12, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/4fe985879e5d4292bdd82491d5ba6412, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/b89ab59d63bd489e878a3ad332d74e07] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp, totalSize=100.9 K 2024-11-07T12:41:58,693 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:58,693 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. files: [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/1e8b21971162423bb6c1d58f6d402e12, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/4fe985879e5d4292bdd82491d5ba6412, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/b89ab59d63bd489e878a3ad332d74e07] 2024-11-07T12:41:58,694 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:41:58,694 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 90c0d1c8af782719d9f8d037e72c2a12/B is initiating minor compaction (all files) 2024-11-07T12:41:58,694 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 90c0d1c8af782719d9f8d037e72c2a12/B in TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:58,694 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/16fa37c0370e4a52a7ffa9ed5907ac7e, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/0e72c86d058a456ea665880421fcdb14, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/5847bb74384a456495abbf3eb36c3603] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp, totalSize=36.8 K 2024-11-07T12:41:58,694 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e8b21971162423bb6c1d58f6d402e12, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1730983317490 2024-11-07T12:41:58,695 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 16fa37c0370e4a52a7ffa9ed5907ac7e, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1730983317490 2024-11-07T12:41:58,695 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4fe985879e5d4292bdd82491d5ba6412, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1730983317677 2024-11-07T12:41:58,695 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e72c86d058a456ea665880421fcdb14, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1730983317677 2024-11-07T12:41:58,695 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting b89ab59d63bd489e878a3ad332d74e07, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1730983318335 2024-11-07T12:41:58,695 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 5847bb74384a456495abbf3eb36c3603, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1730983318337 2024-11-07T12:41:58,707 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=90c0d1c8af782719d9f8d037e72c2a12] 2024-11-07T12:41:58,719 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 90c0d1c8af782719d9f8d037e72c2a12#B#compaction#181 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:41:58,719 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/e07bbcd355a84d71a6436bb44467940a is 50, key is test_row_0/B:col10/1730983318337/Put/seqid=0 2024-11-07T12:41:58,725 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411077e98f111613443238f8af5bbd5be4efb_90c0d1c8af782719d9f8d037e72c2a12 store=[table=TestAcidGuarantees family=A region=90c0d1c8af782719d9f8d037e72c2a12] 2024-11-07T12:41:58,727 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411077e98f111613443238f8af5bbd5be4efb_90c0d1c8af782719d9f8d037e72c2a12, store=[table=TestAcidGuarantees family=A region=90c0d1c8af782719d9f8d037e72c2a12] 2024-11-07T12:41:58,727 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411077e98f111613443238f8af5bbd5be4efb_90c0d1c8af782719d9f8d037e72c2a12 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=90c0d1c8af782719d9f8d037e72c2a12] 2024-11-07T12:41:58,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-07T12:41:58,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742047_1223 (size=4469) 2024-11-07T12:41:58,786 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 90c0d1c8af782719d9f8d037e72c2a12#A#compaction#180 average throughput is 0.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:41:58,787 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/0b14dc3d2cd249628fcad449d98bee8a is 175, key is test_row_0/A:col10/1730983318337/Put/seqid=0 2024-11-07T12:41:58,801 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-07T12:41:58,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:58,802 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing 90c0d1c8af782719d9f8d037e72c2a12 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-07T12:41:58,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 90c0d1c8af782719d9f8d037e72c2a12, store=A 2024-11-07T12:41:58,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:58,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 90c0d1c8af782719d9f8d037e72c2a12, store=B 2024-11-07T12:41:58,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:58,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 90c0d1c8af782719d9f8d037e72c2a12, store=C 2024-11-07T12:41:58,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:41:58,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742048_1224 (size=13221) 2024-11-07T12:41:58,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742049_1225 (size=32175) 2024-11-07T12:41:58,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411073b9d9479a10e4761b2801f450a108b2c_90c0d1c8af782719d9f8d037e72c2a12 is 50, key is test_row_0/A:col10/1730983318504/Put/seqid=0 2024-11-07T12:41:58,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:41:58,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. as already flushing 2024-11-07T12:41:58,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742050_1226 (size=12454) 2024-11-07T12:41:58,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:41:58,882 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411073b9d9479a10e4761b2801f450a108b2c_90c0d1c8af782719d9f8d037e72c2a12 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411073b9d9479a10e4761b2801f450a108b2c_90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:41:58,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/6fb43dab4cff4c84bd0d96cf722fae96, store: [table=TestAcidGuarantees family=A region=90c0d1c8af782719d9f8d037e72c2a12] 2024-11-07T12:41:58,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/6fb43dab4cff4c84bd0d96cf722fae96 is 175, key is test_row_0/A:col10/1730983318504/Put/seqid=0 2024-11-07T12:41:58,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:58,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43038 deadline: 1730983378887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:58,896 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:58,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:58,896 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:58,896 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:58,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43062 deadline: 1730983378888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:58,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:58,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:58,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:58,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:58,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43000 deadline: 1730983378889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:58,897 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:58,897 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:58,897 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:58,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742051_1227 (size=31255) 2024-11-07T12:41:58,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-07T12:41:59,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:59,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43038 deadline: 1730983379002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:59,005 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:59,005 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:59,005 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:59,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:59,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43062 deadline: 1730983379002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:59,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:59,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:59,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:59,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:59,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43000 deadline: 1730983379002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:59,006 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:59,006 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:59,006 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:59,166 INFO [Thread-600 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-600. 2024-11-07T12:41:59,166 DEBUG [Thread-600 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:41:59,166 DEBUG [Thread-600 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:41:59,166 DEBUG [Thread-600 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:41:59,166 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:41:59,168 INFO [Thread-598 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-598. 2024-11-07T12:41:59,168 DEBUG [Thread-598 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:41:59,168 DEBUG [Thread-598 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:41:59,168 INFO [Thread-604 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-604. 2024-11-07T12:41:59,168 DEBUG [Thread-604 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:41:59,168 DEBUG [Thread-604 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:41:59,168 DEBUG [Thread-598 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:41:59,169 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:41:59,169 DEBUG [Thread-604 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:41:59,172 INFO [Thread-602 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-602. 2024-11-07T12:41:59,172 DEBUG [Thread-602 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:41:59,172 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:41:59,172 DEBUG [Thread-602 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:41:59,172 DEBUG [Thread-602 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:41:59,173 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:41:59,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:59,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:59,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43062 deadline: 1730983379212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:59,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43038 deadline: 1730983379212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:59,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:59,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:59,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:59,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:59,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43000 deadline: 1730983379213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:59,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:59,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:59,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:59,214 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:59,214 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:59,215 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:59,223 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/e07bbcd355a84d71a6436bb44467940a as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/e07bbcd355a84d71a6436bb44467940a 2024-11-07T12:41:59,236 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 90c0d1c8af782719d9f8d037e72c2a12/B of 90c0d1c8af782719d9f8d037e72c2a12 into e07bbcd355a84d71a6436bb44467940a(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:41:59,236 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 90c0d1c8af782719d9f8d037e72c2a12: 2024-11-07T12:41:59,236 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., storeName=90c0d1c8af782719d9f8d037e72c2a12/B, priority=13, startTime=1730983318692; duration=0sec 2024-11-07T12:41:59,236 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:41:59,237 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 90c0d1c8af782719d9f8d037e72c2a12:B 2024-11-07T12:41:59,237 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:41:59,237 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/0b14dc3d2cd249628fcad449d98bee8a as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/0b14dc3d2cd249628fcad449d98bee8a 2024-11-07T12:41:59,238 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:41:59,238 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 90c0d1c8af782719d9f8d037e72c2a12/C is initiating minor compaction (all files) 2024-11-07T12:41:59,238 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 90c0d1c8af782719d9f8d037e72c2a12/C in TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:41:59,238 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/f702b6d4c1e04058bfb890407f6c1073, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/0adf79712f2f4da08d9e75e8a8e06d4a, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/dcfb05bb7fb64180a00ea8b4781f075c] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp, totalSize=36.8 K 2024-11-07T12:41:59,238 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting f702b6d4c1e04058bfb890407f6c1073, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1730983317490 2024-11-07T12:41:59,239 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 0adf79712f2f4da08d9e75e8a8e06d4a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1730983317677 2024-11-07T12:41:59,240 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting dcfb05bb7fb64180a00ea8b4781f075c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1730983318337 2024-11-07T12:41:59,243 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 90c0d1c8af782719d9f8d037e72c2a12/A of 90c0d1c8af782719d9f8d037e72c2a12 into 0b14dc3d2cd249628fcad449d98bee8a(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:41:59,243 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 90c0d1c8af782719d9f8d037e72c2a12: 2024-11-07T12:41:59,243 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., storeName=90c0d1c8af782719d9f8d037e72c2a12/A, priority=13, startTime=1730983318691; duration=0sec 2024-11-07T12:41:59,243 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:41:59,243 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 90c0d1c8af782719d9f8d037e72c2a12:A 2024-11-07T12:41:59,251 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 90c0d1c8af782719d9f8d037e72c2a12#C#compaction#183 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:41:59,251 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/1620bf948273412bb69509119b9874e1 is 50, key is test_row_0/C:col10/1730983318337/Put/seqid=0 2024-11-07T12:41:59,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742052_1228 (size=13221) 2024-11-07T12:41:59,265 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/1620bf948273412bb69509119b9874e1 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/1620bf948273412bb69509119b9874e1 2024-11-07T12:41:59,270 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 90c0d1c8af782719d9f8d037e72c2a12/C of 90c0d1c8af782719d9f8d037e72c2a12 into 1620bf948273412bb69509119b9874e1(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:41:59,270 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 90c0d1c8af782719d9f8d037e72c2a12: 2024-11-07T12:41:59,270 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., storeName=90c0d1c8af782719d9f8d037e72c2a12/C, priority=13, startTime=1730983318692; duration=0sec 2024-11-07T12:41:59,270 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:41:59,270 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 90c0d1c8af782719d9f8d037e72c2a12:C 2024-11-07T12:41:59,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-07T12:41:59,317 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=433, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/6fb43dab4cff4c84bd0d96cf722fae96 2024-11-07T12:41:59,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/311c29ae83fc464188254039346f8434 is 50, key is test_row_0/B:col10/1730983318504/Put/seqid=0 2024-11-07T12:41:59,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742053_1229 (size=12301) 2024-11-07T12:41:59,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:59,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:59,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:41:59,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43062 deadline: 1730983379522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:59,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43038 deadline: 1730983379522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:59,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43000 deadline: 1730983379522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:41:59,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:59,523 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:59,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:59,524 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:59,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:59,524 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:59,524 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:59,524 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:41:59,524 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:41:59,766 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/311c29ae83fc464188254039346f8434 2024-11-07T12:41:59,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/9aecb318dfb44ff89abc699335b8b87e is 50, key is test_row_0/C:col10/1730983318504/Put/seqid=0 2024-11-07T12:41:59,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742054_1230 (size=12301) 2024-11-07T12:41:59,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-07T12:42:00,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:00,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:00,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:00,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43000 deadline: 1730983380032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:00,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43062 deadline: 1730983380032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:00,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43038 deadline: 1730983380032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:00,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:00,033 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:00,033 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:00,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:00,033 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:00,033 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:00,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:00,033 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:00,033 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:00,178 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/9aecb318dfb44ff89abc699335b8b87e 2024-11-07T12:42:00,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/6fb43dab4cff4c84bd0d96cf722fae96 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/6fb43dab4cff4c84bd0d96cf722fae96 2024-11-07T12:42:00,192 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/6fb43dab4cff4c84bd0d96cf722fae96, entries=150, sequenceid=433, filesize=30.5 K 2024-11-07T12:42:00,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/311c29ae83fc464188254039346f8434 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/311c29ae83fc464188254039346f8434 2024-11-07T12:42:00,202 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/311c29ae83fc464188254039346f8434, entries=150, sequenceid=433, filesize=12.0 K 2024-11-07T12:42:00,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/9aecb318dfb44ff89abc699335b8b87e as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/9aecb318dfb44ff89abc699335b8b87e 2024-11-07T12:42:00,213 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/9aecb318dfb44ff89abc699335b8b87e, entries=150, sequenceid=433, filesize=12.0 K 2024-11-07T12:42:00,217 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 90c0d1c8af782719d9f8d037e72c2a12 in 1415ms, sequenceid=433, compaction requested=false 2024-11-07T12:42:00,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for 90c0d1c8af782719d9f8d037e72c2a12: 2024-11-07T12:42:00,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:42:00,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-07T12:42:00,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-07T12:42:00,221 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-07T12:42:00,221 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5710 sec 2024-11-07T12:42:00,224 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 1.5790 sec 2024-11-07T12:42:00,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-07T12:42:00,792 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:42:01,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:42:01,044 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 90c0d1c8af782719d9f8d037e72c2a12 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-07T12:42:01,044 INFO [Thread-589 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-589. 2024-11-07T12:42:01,044 DEBUG [Thread-589 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:42:01,044 DEBUG [Thread-589 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:42:01,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 90c0d1c8af782719d9f8d037e72c2a12, store=A 2024-11-07T12:42:01,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:01,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 90c0d1c8af782719d9f8d037e72c2a12, store=B 2024-11-07T12:42:01,044 DEBUG [Thread-589 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:42:01,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:01,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 90c0d1c8af782719d9f8d037e72c2a12, store=C 2024-11-07T12:42:01,044 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:42:01,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:01,046 INFO [Thread-591 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-591. 2024-11-07T12:42:01,046 DEBUG [Thread-591 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:42:01,046 DEBUG [Thread-591 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:42:01,046 DEBUG [Thread-591 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:42:01,046 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:42:01,054 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110794c3d5ef8d8c498eaa4b94fdc8e96535_90c0d1c8af782719d9f8d037e72c2a12 is 50, key is test_row_0/A:col10/1730983321042/Put/seqid=0 2024-11-07T12:42:01,056 INFO [Thread-595 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-595. 2024-11-07T12:42:01,056 DEBUG [Thread-595 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:42:01,056 DEBUG [Thread-595 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:42:01,056 DEBUG [Thread-595 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:42:01,056 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:42:01,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742055_1231 (size=12454) 2024-11-07T12:42:01,460 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:01,465 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110794c3d5ef8d8c498eaa4b94fdc8e96535_90c0d1c8af782719d9f8d037e72c2a12 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110794c3d5ef8d8c498eaa4b94fdc8e96535_90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:42:01,466 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/54d6baf02f3342ad985ab85870555e86, store: [table=TestAcidGuarantees family=A region=90c0d1c8af782719d9f8d037e72c2a12] 2024-11-07T12:42:01,467 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/54d6baf02f3342ad985ab85870555e86 is 175, key is test_row_0/A:col10/1730983321042/Put/seqid=0 2024-11-07T12:42:01,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742056_1232 (size=31255) 2024-11-07T12:42:01,472 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=450, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/54d6baf02f3342ad985ab85870555e86 2024-11-07T12:42:01,482 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/40e9a49fcba64b6a8479931862214f29 is 50, key is test_row_0/B:col10/1730983321042/Put/seqid=0 2024-11-07T12:42:01,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742057_1233 (size=12301) 2024-11-07T12:42:01,894 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/40e9a49fcba64b6a8479931862214f29 2024-11-07T12:42:01,903 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/8320042fab754a12a320a3a608487054 is 50, key is test_row_0/C:col10/1730983321042/Put/seqid=0 2024-11-07T12:42:01,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742058_1234 (size=12301) 2024-11-07T12:42:02,308 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/8320042fab754a12a320a3a608487054 2024-11-07T12:42:02,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/54d6baf02f3342ad985ab85870555e86 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/54d6baf02f3342ad985ab85870555e86 2024-11-07T12:42:02,317 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/54d6baf02f3342ad985ab85870555e86, entries=150, sequenceid=450, filesize=30.5 K 2024-11-07T12:42:02,318 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/40e9a49fcba64b6a8479931862214f29 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/40e9a49fcba64b6a8479931862214f29 2024-11-07T12:42:02,323 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/40e9a49fcba64b6a8479931862214f29, entries=150, sequenceid=450, filesize=12.0 K 2024-11-07T12:42:02,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/8320042fab754a12a320a3a608487054 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/8320042fab754a12a320a3a608487054 2024-11-07T12:42:02,328 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/8320042fab754a12a320a3a608487054, entries=150, sequenceid=450, filesize=12.0 K 2024-11-07T12:42:02,329 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=6.71 KB/6870 for 90c0d1c8af782719d9f8d037e72c2a12 in 1284ms, sequenceid=450, compaction requested=true 2024-11-07T12:42:02,329 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 90c0d1c8af782719d9f8d037e72c2a12: 2024-11-07T12:42:02,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 90c0d1c8af782719d9f8d037e72c2a12:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:42:02,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:42:02,329 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:42:02,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 90c0d1c8af782719d9f8d037e72c2a12:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:42:02,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:42:02,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 90c0d1c8af782719d9f8d037e72c2a12:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:42:02,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:42:02,329 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:42:02,330 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94685 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:42:02,330 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): 90c0d1c8af782719d9f8d037e72c2a12/A is initiating minor compaction (all files) 2024-11-07T12:42:02,330 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 90c0d1c8af782719d9f8d037e72c2a12/A in TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:42:02,330 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/0b14dc3d2cd249628fcad449d98bee8a, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/6fb43dab4cff4c84bd0d96cf722fae96, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/54d6baf02f3342ad985ab85870555e86] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp, totalSize=92.5 K 2024-11-07T12:42:02,330 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:42:02,330 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:42:02,330 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. files: [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/0b14dc3d2cd249628fcad449d98bee8a, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/6fb43dab4cff4c84bd0d96cf722fae96, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/54d6baf02f3342ad985ab85870555e86] 2024-11-07T12:42:02,330 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 90c0d1c8af782719d9f8d037e72c2a12/B is initiating minor compaction (all files) 2024-11-07T12:42:02,330 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 90c0d1c8af782719d9f8d037e72c2a12/B in TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:42:02,330 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/e07bbcd355a84d71a6436bb44467940a, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/311c29ae83fc464188254039346f8434, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/40e9a49fcba64b6a8479931862214f29] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp, totalSize=36.9 K 2024-11-07T12:42:02,330 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b14dc3d2cd249628fcad449d98bee8a, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1730983318337 2024-11-07T12:42:02,331 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting e07bbcd355a84d71a6436bb44467940a, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1730983318337 2024-11-07T12:42:02,331 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6fb43dab4cff4c84bd0d96cf722fae96, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1730983318504 2024-11-07T12:42:02,331 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 311c29ae83fc464188254039346f8434, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1730983318504 2024-11-07T12:42:02,331 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54d6baf02f3342ad985ab85870555e86, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=450, earliestPutTs=1730983318855 2024-11-07T12:42:02,332 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 40e9a49fcba64b6a8479931862214f29, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=450, earliestPutTs=1730983318855 2024-11-07T12:42:02,342 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 90c0d1c8af782719d9f8d037e72c2a12#B#compaction#189 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:42:02,342 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/09209ff30ed64e03b684aeb35162d0cf is 50, key is test_row_0/B:col10/1730983321042/Put/seqid=0 2024-11-07T12:42:02,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742059_1235 (size=13323) 2024-11-07T12:42:02,354 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=90c0d1c8af782719d9f8d037e72c2a12] 2024-11-07T12:42:02,356 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024110768c55c732efa4f1182d2c5474a5dadb6_90c0d1c8af782719d9f8d037e72c2a12 store=[table=TestAcidGuarantees family=A region=90c0d1c8af782719d9f8d037e72c2a12] 2024-11-07T12:42:02,386 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024110768c55c732efa4f1182d2c5474a5dadb6_90c0d1c8af782719d9f8d037e72c2a12, store=[table=TestAcidGuarantees family=A region=90c0d1c8af782719d9f8d037e72c2a12] 2024-11-07T12:42:02,386 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110768c55c732efa4f1182d2c5474a5dadb6_90c0d1c8af782719d9f8d037e72c2a12 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=90c0d1c8af782719d9f8d037e72c2a12] 2024-11-07T12:42:02,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742060_1236 (size=4469) 2024-11-07T12:42:02,394 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 90c0d1c8af782719d9f8d037e72c2a12#A#compaction#190 average throughput is 0.61 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:42:02,395 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/a9a626c1b0d942b2a6191b0745107b55 is 175, key is test_row_0/A:col10/1730983321042/Put/seqid=0 2024-11-07T12:42:02,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742061_1237 (size=32277) 2024-11-07T12:42:02,413 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/a9a626c1b0d942b2a6191b0745107b55 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/a9a626c1b0d942b2a6191b0745107b55 2024-11-07T12:42:02,432 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 90c0d1c8af782719d9f8d037e72c2a12/A of 90c0d1c8af782719d9f8d037e72c2a12 into a9a626c1b0d942b2a6191b0745107b55(size=31.5 K), total size for store is 31.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:42:02,432 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 90c0d1c8af782719d9f8d037e72c2a12: 2024-11-07T12:42:02,432 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., storeName=90c0d1c8af782719d9f8d037e72c2a12/A, priority=13, startTime=1730983322329; duration=0sec 2024-11-07T12:42:02,433 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:42:02,433 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 90c0d1c8af782719d9f8d037e72c2a12:A 2024-11-07T12:42:02,433 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:42:02,434 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:42:02,434 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): 90c0d1c8af782719d9f8d037e72c2a12/C is initiating minor compaction (all files) 2024-11-07T12:42:02,434 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 90c0d1c8af782719d9f8d037e72c2a12/C in TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:42:02,435 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/1620bf948273412bb69509119b9874e1, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/9aecb318dfb44ff89abc699335b8b87e, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/8320042fab754a12a320a3a608487054] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp, totalSize=36.9 K 2024-11-07T12:42:02,436 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1620bf948273412bb69509119b9874e1, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1730983318337 2024-11-07T12:42:02,437 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9aecb318dfb44ff89abc699335b8b87e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1730983318504 2024-11-07T12:42:02,438 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8320042fab754a12a320a3a608487054, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=450, earliestPutTs=1730983318855 2024-11-07T12:42:02,467 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 90c0d1c8af782719d9f8d037e72c2a12#C#compaction#191 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:42:02,468 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/ef89e9c3a5db4368b1b9c567de8d25b2 is 50, key is test_row_0/C:col10/1730983321042/Put/seqid=0 2024-11-07T12:42:02,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742062_1238 (size=13323) 2024-11-07T12:42:02,758 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/09209ff30ed64e03b684aeb35162d0cf as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/09209ff30ed64e03b684aeb35162d0cf 2024-11-07T12:42:02,765 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 90c0d1c8af782719d9f8d037e72c2a12/B of 90c0d1c8af782719d9f8d037e72c2a12 into 09209ff30ed64e03b684aeb35162d0cf(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:42:02,765 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 90c0d1c8af782719d9f8d037e72c2a12: 2024-11-07T12:42:02,765 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., storeName=90c0d1c8af782719d9f8d037e72c2a12/B, priority=13, startTime=1730983322329; duration=0sec 2024-11-07T12:42:02,765 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:42:02,765 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 90c0d1c8af782719d9f8d037e72c2a12:B 2024-11-07T12:42:02,886 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/ef89e9c3a5db4368b1b9c567de8d25b2 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/ef89e9c3a5db4368b1b9c567de8d25b2 2024-11-07T12:42:02,892 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 90c0d1c8af782719d9f8d037e72c2a12/C of 90c0d1c8af782719d9f8d037e72c2a12 into ef89e9c3a5db4368b1b9c567de8d25b2(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:42:02,892 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 90c0d1c8af782719d9f8d037e72c2a12: 2024-11-07T12:42:02,892 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., storeName=90c0d1c8af782719d9f8d037e72c2a12/C, priority=13, startTime=1730983322329; duration=0sec 2024-11-07T12:42:02,893 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:42:02,893 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 90c0d1c8af782719d9f8d037e72c2a12:C 2024-11-07T12:42:04,273 INFO [Thread-587 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-587. 2024-11-07T12:42:04,273 DEBUG [Thread-587 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:42:04,273 DEBUG [Thread-587 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:42:04,273 DEBUG [Thread-587 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:42:04,273 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:42:04,283 INFO [Thread-593 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-593. 2024-11-07T12:42:04,283 DEBUG [Thread-593 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:42:04,283 DEBUG [Thread-593 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:42:04,283 DEBUG [Thread-593 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:42:04,283 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:42:04,283 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-07T12:42:04,283 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 26 2024-11-07T12:42:04,283 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 108 2024-11-07T12:42:04,283 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 82 2024-11-07T12:42:04,283 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 33 2024-11-07T12:42:04,283 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 108 2024-11-07T12:42:04,283 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-07T12:42:04,283 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1926 2024-11-07T12:42:04,283 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1912 2024-11-07T12:42:04,283 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-07T12:42:04,283 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 871 2024-11-07T12:42:04,283 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2613 rows 2024-11-07T12:42:04,283 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 871 2024-11-07T12:42:04,283 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2611 rows 2024-11-07T12:42:04,283 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-07T12:42:04,284 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T12:42:04,284 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:42:04,284 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:42:04,284 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:42:04,284 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-07T12:42:04,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-07T12:42:04,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-07T12:42:04,287 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730983324287"}]},"ts":"1730983324287"} 2024-11-07T12:42:04,289 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-07T12:42:04,289 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-07T12:42:04,289 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-07T12:42:04,291 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=90c0d1c8af782719d9f8d037e72c2a12, UNASSIGN}] 2024-11-07T12:42:04,291 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=90c0d1c8af782719d9f8d037e72c2a12, UNASSIGN 2024-11-07T12:42:04,292 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=90c0d1c8af782719d9f8d037e72c2a12, regionState=CLOSING, regionLocation=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:04,293 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=90c0d1c8af782719d9f8d037e72c2a12, UNASSIGN because future has completed 2024-11-07T12:42:04,293 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-07T12:42:04,293 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; CloseRegionProcedure 90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217}] 2024-11-07T12:42:04,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-07T12:42:04,446 INFO [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(122): Close 90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:42:04,446 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-07T12:42:04,446 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1722): Closing 90c0d1c8af782719d9f8d037e72c2a12, disabling compactions & flushes 2024-11-07T12:42:04,446 INFO [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:42:04,446 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:42:04,446 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. after waiting 0 ms 2024-11-07T12:42:04,446 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:42:04,446 INFO [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(2902): Flushing 90c0d1c8af782719d9f8d037e72c2a12 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-07T12:42:04,446 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 90c0d1c8af782719d9f8d037e72c2a12, store=A 2024-11-07T12:42:04,447 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:04,447 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 90c0d1c8af782719d9f8d037e72c2a12, store=B 2024-11-07T12:42:04,447 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:04,447 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 90c0d1c8af782719d9f8d037e72c2a12, store=C 2024-11-07T12:42:04,447 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:04,454 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411073f30dd36805c4c249565739bd21cc858_90c0d1c8af782719d9f8d037e72c2a12 is 50, key is test_row_0/A:col10/1730983324282/Put/seqid=0 2024-11-07T12:42:04,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742063_1239 (size=9914) 2024-11-07T12:42:04,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-07T12:42:04,860 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:04,864 INFO [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411073f30dd36805c4c249565739bd21cc858_90c0d1c8af782719d9f8d037e72c2a12 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411073f30dd36805c4c249565739bd21cc858_90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:42:04,865 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/ae8bfe27110f4dd8b2c78c0432d8c19c, store: [table=TestAcidGuarantees family=A region=90c0d1c8af782719d9f8d037e72c2a12] 2024-11-07T12:42:04,866 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/ae8bfe27110f4dd8b2c78c0432d8c19c is 175, key is test_row_0/A:col10/1730983324282/Put/seqid=0 2024-11-07T12:42:04,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742064_1240 (size=22561) 2024-11-07T12:42:04,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-07T12:42:05,271 INFO [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=459, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/ae8bfe27110f4dd8b2c78c0432d8c19c 2024-11-07T12:42:05,278 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/ba2a13013e904eaf9025f1a9e38d362a is 50, key is test_row_0/B:col10/1730983324282/Put/seqid=0 2024-11-07T12:42:05,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742065_1241 (size=9857) 2024-11-07T12:42:05,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-07T12:42:05,682 INFO [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/ba2a13013e904eaf9025f1a9e38d362a 2024-11-07T12:42:05,691 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/f0ef499135c04c13803cfc87ff77c196 is 50, key is test_row_0/C:col10/1730983324282/Put/seqid=0 2024-11-07T12:42:05,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742066_1242 (size=9857) 2024-11-07T12:42:06,095 INFO [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/f0ef499135c04c13803cfc87ff77c196 2024-11-07T12:42:06,099 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/A/ae8bfe27110f4dd8b2c78c0432d8c19c as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/ae8bfe27110f4dd8b2c78c0432d8c19c 2024-11-07T12:42:06,103 INFO [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/ae8bfe27110f4dd8b2c78c0432d8c19c, entries=100, sequenceid=459, filesize=22.0 K 2024-11-07T12:42:06,104 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/B/ba2a13013e904eaf9025f1a9e38d362a as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/ba2a13013e904eaf9025f1a9e38d362a 2024-11-07T12:42:06,108 INFO [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/ba2a13013e904eaf9025f1a9e38d362a, entries=100, sequenceid=459, filesize=9.6 K 2024-11-07T12:42:06,109 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/.tmp/C/f0ef499135c04c13803cfc87ff77c196 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/f0ef499135c04c13803cfc87ff77c196 2024-11-07T12:42:06,113 INFO [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/f0ef499135c04c13803cfc87ff77c196, entries=100, sequenceid=459, filesize=9.6 K 2024-11-07T12:42:06,114 INFO [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 90c0d1c8af782719d9f8d037e72c2a12 in 1668ms, sequenceid=459, compaction requested=false 2024-11-07T12:42:06,115 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/e57a80a0dfa84e38bc21cb6118c2e7b3, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/91f8ed42a5fc4d7db673a90a75de4aa6, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/101b717dc16c43ddbd91cf876eae76c3, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/fa30f245fa58420bacea55f4c1e8ba56, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/821b1f1bee6241a99cf8c7bb1e2a7ed6, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/61dbde97014940568754ffb791b95fa3, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/d4ea87bc8bcd475fbd43e3f8e7fd4368, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/786576761519488483e13b8b2c1e6a81, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/92f6310dadfc40a6a1c426a546001085, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/61ca6416d8d54eb4bcf3f4a893bcb846, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/ecd77c8d37cb4d3bbd6a358f36a947b5, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/a9000c06a877406c981aee490c136ba8, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/4a5602684f7342b2a0a32d26ce121c3a, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/7757eeeeef414dd5a23bcc6569148a3c, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/a0ee187b1da8408d9d73c194d72b4e01, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/0ddbae73e35141c39ae9f8e6372a3d3f, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/2d9225edf52b4408a60fdf06c3aaef71, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/ac0349aca1bf4e05a1b5886b70fb6cca, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/33c51829bec8419ba5996dde87d66d22, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/74723be07b174b008693cfd06571d16e, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/dd88931613b04986b4630968c2713385, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/d100ce85e2a94aa1b4f7303f1c43016b, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/011f62f1ad644c35b58c0d47491725ac, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/87a2ffdf4a22414e84ef6cb022490537, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/1e8b21971162423bb6c1d58f6d402e12, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/4fe985879e5d4292bdd82491d5ba6412, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/b89ab59d63bd489e878a3ad332d74e07, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/0b14dc3d2cd249628fcad449d98bee8a, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/6fb43dab4cff4c84bd0d96cf722fae96, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/54d6baf02f3342ad985ab85870555e86] to archive 2024-11-07T12:42:06,116 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T12:42:06,117 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/e57a80a0dfa84e38bc21cb6118c2e7b3 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/e57a80a0dfa84e38bc21cb6118c2e7b3 2024-11-07T12:42:06,118 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/91f8ed42a5fc4d7db673a90a75de4aa6 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/91f8ed42a5fc4d7db673a90a75de4aa6 2024-11-07T12:42:06,120 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/101b717dc16c43ddbd91cf876eae76c3 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/101b717dc16c43ddbd91cf876eae76c3 2024-11-07T12:42:06,121 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/fa30f245fa58420bacea55f4c1e8ba56 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/fa30f245fa58420bacea55f4c1e8ba56 2024-11-07T12:42:06,122 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/821b1f1bee6241a99cf8c7bb1e2a7ed6 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/821b1f1bee6241a99cf8c7bb1e2a7ed6 2024-11-07T12:42:06,123 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/61dbde97014940568754ffb791b95fa3 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/61dbde97014940568754ffb791b95fa3 2024-11-07T12:42:06,125 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/d4ea87bc8bcd475fbd43e3f8e7fd4368 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/d4ea87bc8bcd475fbd43e3f8e7fd4368 2024-11-07T12:42:06,126 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/786576761519488483e13b8b2c1e6a81 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/786576761519488483e13b8b2c1e6a81 2024-11-07T12:42:06,127 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/92f6310dadfc40a6a1c426a546001085 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/92f6310dadfc40a6a1c426a546001085 2024-11-07T12:42:06,129 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/61ca6416d8d54eb4bcf3f4a893bcb846 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/61ca6416d8d54eb4bcf3f4a893bcb846 2024-11-07T12:42:06,130 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/ecd77c8d37cb4d3bbd6a358f36a947b5 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/ecd77c8d37cb4d3bbd6a358f36a947b5 2024-11-07T12:42:06,131 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/a9000c06a877406c981aee490c136ba8 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/a9000c06a877406c981aee490c136ba8 2024-11-07T12:42:06,132 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/4a5602684f7342b2a0a32d26ce121c3a to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/4a5602684f7342b2a0a32d26ce121c3a 2024-11-07T12:42:06,133 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/7757eeeeef414dd5a23bcc6569148a3c to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/7757eeeeef414dd5a23bcc6569148a3c 2024-11-07T12:42:06,134 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/a0ee187b1da8408d9d73c194d72b4e01 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/a0ee187b1da8408d9d73c194d72b4e01 2024-11-07T12:42:06,135 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/0ddbae73e35141c39ae9f8e6372a3d3f to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/0ddbae73e35141c39ae9f8e6372a3d3f 2024-11-07T12:42:06,137 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/2d9225edf52b4408a60fdf06c3aaef71 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/2d9225edf52b4408a60fdf06c3aaef71 2024-11-07T12:42:06,138 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/ac0349aca1bf4e05a1b5886b70fb6cca to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/ac0349aca1bf4e05a1b5886b70fb6cca 2024-11-07T12:42:06,139 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/33c51829bec8419ba5996dde87d66d22 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/33c51829bec8419ba5996dde87d66d22 2024-11-07T12:42:06,140 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/74723be07b174b008693cfd06571d16e to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/74723be07b174b008693cfd06571d16e 2024-11-07T12:42:06,141 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/dd88931613b04986b4630968c2713385 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/dd88931613b04986b4630968c2713385 2024-11-07T12:42:06,142 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/d100ce85e2a94aa1b4f7303f1c43016b to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/d100ce85e2a94aa1b4f7303f1c43016b 2024-11-07T12:42:06,144 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/011f62f1ad644c35b58c0d47491725ac to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/011f62f1ad644c35b58c0d47491725ac 2024-11-07T12:42:06,145 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/87a2ffdf4a22414e84ef6cb022490537 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/87a2ffdf4a22414e84ef6cb022490537 2024-11-07T12:42:06,146 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/1e8b21971162423bb6c1d58f6d402e12 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/1e8b21971162423bb6c1d58f6d402e12 2024-11-07T12:42:06,147 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/4fe985879e5d4292bdd82491d5ba6412 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/4fe985879e5d4292bdd82491d5ba6412 2024-11-07T12:42:06,149 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/b89ab59d63bd489e878a3ad332d74e07 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/b89ab59d63bd489e878a3ad332d74e07 2024-11-07T12:42:06,150 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/0b14dc3d2cd249628fcad449d98bee8a to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/0b14dc3d2cd249628fcad449d98bee8a 2024-11-07T12:42:06,152 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/6fb43dab4cff4c84bd0d96cf722fae96 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/6fb43dab4cff4c84bd0d96cf722fae96 2024-11-07T12:42:06,153 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/54d6baf02f3342ad985ab85870555e86 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/54d6baf02f3342ad985ab85870555e86 2024-11-07T12:42:06,155 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/ad2697a5acd64bac8322b00b13969389, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/f52a2415307c48cc980c0c3dcfc4eda3, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/f1437c3c37174f49b4aa4fe48b3098b3, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/ecdc018dd9194b0d8fe83793d52159a8, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/4a3b2d7d850741958a8518dbf7184188, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/9758b955b00946f5bf71155e134ae797, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/079dbb8b9b4d4b769d4525d014244aa8, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/a4dc22bb78734c9ca5ef51d8372b00ef, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/c3c6370ee41c401d9be1b718899b213e, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/3a317eea129b4cb4af883a40293ff241, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/afdd0ce03813446793ef00b990070de2, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/6218b4f9350e4fa8a230d63194b4d469, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/5684dac7279347ce9a65a3b1f952814d, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/ae85411afa064738b4bccd784b031c7a, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/e0f21398f98f49a58406a171ebd94b93, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/c7f6ca93f96544bd91bbc0a4c1c18f56, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/a1b2c69f572941ce81b8c9f43138631f, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/a35776c4f094458d81f13ad96df000f9, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/bbe5c39094634aa3bca8cedef889e925, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/fd3db2b9a1a24a1bb39f0b43dfb029cc, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/0ab7a877c9174fa795ba8cbaed854bdc, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/6401b48fb2c842419dccf7474c63c948, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/5536db3c5ee6483ba192deacacfcf700, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/16fa37c0370e4a52a7ffa9ed5907ac7e, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/5bdd9ae4a196473db727833e65b5e0ae, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/0e72c86d058a456ea665880421fcdb14, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/e07bbcd355a84d71a6436bb44467940a, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/5847bb74384a456495abbf3eb36c3603, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/311c29ae83fc464188254039346f8434, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/40e9a49fcba64b6a8479931862214f29] to archive 2024-11-07T12:42:06,156 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T12:42:06,157 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/ad2697a5acd64bac8322b00b13969389 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/ad2697a5acd64bac8322b00b13969389 2024-11-07T12:42:06,158 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/f52a2415307c48cc980c0c3dcfc4eda3 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/f52a2415307c48cc980c0c3dcfc4eda3 2024-11-07T12:42:06,160 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/f1437c3c37174f49b4aa4fe48b3098b3 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/f1437c3c37174f49b4aa4fe48b3098b3 2024-11-07T12:42:06,161 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/ecdc018dd9194b0d8fe83793d52159a8 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/ecdc018dd9194b0d8fe83793d52159a8 2024-11-07T12:42:06,162 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/4a3b2d7d850741958a8518dbf7184188 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/4a3b2d7d850741958a8518dbf7184188 2024-11-07T12:42:06,163 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/9758b955b00946f5bf71155e134ae797 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/9758b955b00946f5bf71155e134ae797 2024-11-07T12:42:06,165 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/079dbb8b9b4d4b769d4525d014244aa8 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/079dbb8b9b4d4b769d4525d014244aa8 2024-11-07T12:42:06,166 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/a4dc22bb78734c9ca5ef51d8372b00ef to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/a4dc22bb78734c9ca5ef51d8372b00ef 2024-11-07T12:42:06,167 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/c3c6370ee41c401d9be1b718899b213e to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/c3c6370ee41c401d9be1b718899b213e 2024-11-07T12:42:06,168 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/3a317eea129b4cb4af883a40293ff241 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/3a317eea129b4cb4af883a40293ff241 2024-11-07T12:42:06,170 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/afdd0ce03813446793ef00b990070de2 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/afdd0ce03813446793ef00b990070de2 2024-11-07T12:42:06,171 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/6218b4f9350e4fa8a230d63194b4d469 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/6218b4f9350e4fa8a230d63194b4d469 2024-11-07T12:42:06,172 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/5684dac7279347ce9a65a3b1f952814d to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/5684dac7279347ce9a65a3b1f952814d 2024-11-07T12:42:06,173 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/ae85411afa064738b4bccd784b031c7a to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/ae85411afa064738b4bccd784b031c7a 2024-11-07T12:42:06,175 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/e0f21398f98f49a58406a171ebd94b93 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/e0f21398f98f49a58406a171ebd94b93 2024-11-07T12:42:06,176 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/c7f6ca93f96544bd91bbc0a4c1c18f56 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/c7f6ca93f96544bd91bbc0a4c1c18f56 2024-11-07T12:42:06,177 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/a1b2c69f572941ce81b8c9f43138631f to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/a1b2c69f572941ce81b8c9f43138631f 2024-11-07T12:42:06,178 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/a35776c4f094458d81f13ad96df000f9 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/a35776c4f094458d81f13ad96df000f9 2024-11-07T12:42:06,179 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/bbe5c39094634aa3bca8cedef889e925 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/bbe5c39094634aa3bca8cedef889e925 2024-11-07T12:42:06,181 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/fd3db2b9a1a24a1bb39f0b43dfb029cc to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/fd3db2b9a1a24a1bb39f0b43dfb029cc 2024-11-07T12:42:06,182 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/0ab7a877c9174fa795ba8cbaed854bdc to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/0ab7a877c9174fa795ba8cbaed854bdc 2024-11-07T12:42:06,184 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/6401b48fb2c842419dccf7474c63c948 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/6401b48fb2c842419dccf7474c63c948 2024-11-07T12:42:06,185 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/5536db3c5ee6483ba192deacacfcf700 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/5536db3c5ee6483ba192deacacfcf700 2024-11-07T12:42:06,186 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/16fa37c0370e4a52a7ffa9ed5907ac7e to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/16fa37c0370e4a52a7ffa9ed5907ac7e 2024-11-07T12:42:06,187 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/5bdd9ae4a196473db727833e65b5e0ae to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/5bdd9ae4a196473db727833e65b5e0ae 2024-11-07T12:42:06,189 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/0e72c86d058a456ea665880421fcdb14 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/0e72c86d058a456ea665880421fcdb14 2024-11-07T12:42:06,190 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/e07bbcd355a84d71a6436bb44467940a to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/e07bbcd355a84d71a6436bb44467940a 2024-11-07T12:42:06,191 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/5847bb74384a456495abbf3eb36c3603 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/5847bb74384a456495abbf3eb36c3603 2024-11-07T12:42:06,192 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/311c29ae83fc464188254039346f8434 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/311c29ae83fc464188254039346f8434 2024-11-07T12:42:06,195 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/40e9a49fcba64b6a8479931862214f29 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/40e9a49fcba64b6a8479931862214f29 2024-11-07T12:42:06,198 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/47ce9bc6ffe44d3eb1ace53f8d626150, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/c50b605f280044f59c4ae0341ced5511, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/191f690638bf42758c47af0fbc73bc03, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/1f5534c1ceae4975a971ebc01e171453, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/8502002f71db4430a42f696609efdac6, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/66c2ccad37e34d83b3fef5712de17085, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/00575456d4cf4570b87759b688c27c24, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/aacbd0d5d673448e9a8076c06af80705, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/8818d0852d0544f3aa4b676dd78162d2, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/f29b0ee4db864c799f8cf958f2e894e0, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/f1fa25483a81465d80dc8c1d05d39e4d, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/1abb5d6c5ca54a1abe404f1a3c0848e2, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/fe26956dc6e14c88b9c9d52c23372a57, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/72c8293d7d8c4e8aa4e496516e8eceb4, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/76f0c164606340238fccfc5068df4d24, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/adc90fca257d4c0a961c844b2eb1ac58, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/ef09e7c6663341d997fcb08c20339fd7, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/ee5d451d571b42f984324af0d399c642, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/c740cfd43765441f86fd60e108b4043c, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/86f06de6234249afb55147aa826ae715, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/5137832d790a494aa2c6df7b411f8380, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/b921047a98044531af923a9231c3f798, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/1c04e825af2043ed972d46ce385a545c, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/f702b6d4c1e04058bfb890407f6c1073, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/2ad36ff40b45481dac6e548f6b7f42fd, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/0adf79712f2f4da08d9e75e8a8e06d4a, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/1620bf948273412bb69509119b9874e1, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/dcfb05bb7fb64180a00ea8b4781f075c, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/9aecb318dfb44ff89abc699335b8b87e, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/8320042fab754a12a320a3a608487054] to archive 2024-11-07T12:42:06,199 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T12:42:06,200 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/47ce9bc6ffe44d3eb1ace53f8d626150 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/47ce9bc6ffe44d3eb1ace53f8d626150 2024-11-07T12:42:06,201 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/c50b605f280044f59c4ae0341ced5511 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/c50b605f280044f59c4ae0341ced5511 2024-11-07T12:42:06,202 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/191f690638bf42758c47af0fbc73bc03 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/191f690638bf42758c47af0fbc73bc03 2024-11-07T12:42:06,204 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/1f5534c1ceae4975a971ebc01e171453 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/1f5534c1ceae4975a971ebc01e171453 2024-11-07T12:42:06,205 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/8502002f71db4430a42f696609efdac6 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/8502002f71db4430a42f696609efdac6 2024-11-07T12:42:06,206 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/66c2ccad37e34d83b3fef5712de17085 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/66c2ccad37e34d83b3fef5712de17085 2024-11-07T12:42:06,207 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/00575456d4cf4570b87759b688c27c24 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/00575456d4cf4570b87759b688c27c24 2024-11-07T12:42:06,208 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/aacbd0d5d673448e9a8076c06af80705 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/aacbd0d5d673448e9a8076c06af80705 2024-11-07T12:42:06,209 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/8818d0852d0544f3aa4b676dd78162d2 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/8818d0852d0544f3aa4b676dd78162d2 2024-11-07T12:42:06,210 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/f29b0ee4db864c799f8cf958f2e894e0 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/f29b0ee4db864c799f8cf958f2e894e0 2024-11-07T12:42:06,211 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/f1fa25483a81465d80dc8c1d05d39e4d to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/f1fa25483a81465d80dc8c1d05d39e4d 2024-11-07T12:42:06,212 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/1abb5d6c5ca54a1abe404f1a3c0848e2 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/1abb5d6c5ca54a1abe404f1a3c0848e2 2024-11-07T12:42:06,213 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/fe26956dc6e14c88b9c9d52c23372a57 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/fe26956dc6e14c88b9c9d52c23372a57 2024-11-07T12:42:06,214 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/72c8293d7d8c4e8aa4e496516e8eceb4 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/72c8293d7d8c4e8aa4e496516e8eceb4 2024-11-07T12:42:06,215 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/76f0c164606340238fccfc5068df4d24 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/76f0c164606340238fccfc5068df4d24 2024-11-07T12:42:06,216 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/adc90fca257d4c0a961c844b2eb1ac58 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/adc90fca257d4c0a961c844b2eb1ac58 2024-11-07T12:42:06,217 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/ef09e7c6663341d997fcb08c20339fd7 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/ef09e7c6663341d997fcb08c20339fd7 2024-11-07T12:42:06,218 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/ee5d451d571b42f984324af0d399c642 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/ee5d451d571b42f984324af0d399c642 2024-11-07T12:42:06,219 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/c740cfd43765441f86fd60e108b4043c to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/c740cfd43765441f86fd60e108b4043c 2024-11-07T12:42:06,221 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/86f06de6234249afb55147aa826ae715 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/86f06de6234249afb55147aa826ae715 2024-11-07T12:42:06,221 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/5137832d790a494aa2c6df7b411f8380 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/5137832d790a494aa2c6df7b411f8380 2024-11-07T12:42:06,222 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/b921047a98044531af923a9231c3f798 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/b921047a98044531af923a9231c3f798 2024-11-07T12:42:06,224 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/1c04e825af2043ed972d46ce385a545c to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/1c04e825af2043ed972d46ce385a545c 2024-11-07T12:42:06,225 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/f702b6d4c1e04058bfb890407f6c1073 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/f702b6d4c1e04058bfb890407f6c1073 2024-11-07T12:42:06,225 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/2ad36ff40b45481dac6e548f6b7f42fd to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/2ad36ff40b45481dac6e548f6b7f42fd 2024-11-07T12:42:06,226 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/0adf79712f2f4da08d9e75e8a8e06d4a to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/0adf79712f2f4da08d9e75e8a8e06d4a 2024-11-07T12:42:06,227 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/1620bf948273412bb69509119b9874e1 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/1620bf948273412bb69509119b9874e1 2024-11-07T12:42:06,228 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/dcfb05bb7fb64180a00ea8b4781f075c to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/dcfb05bb7fb64180a00ea8b4781f075c 2024-11-07T12:42:06,229 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/9aecb318dfb44ff89abc699335b8b87e to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/9aecb318dfb44ff89abc699335b8b87e 2024-11-07T12:42:06,230 DEBUG [StoreCloser-TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/8320042fab754a12a320a3a608487054 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/8320042fab754a12a320a3a608487054 2024-11-07T12:42:06,235 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/recovered.edits/462.seqid, newMaxSeqId=462, maxSeqId=4 2024-11-07T12:42:06,237 INFO [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. 2024-11-07T12:42:06,237 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1676): Region close journal for 90c0d1c8af782719d9f8d037e72c2a12: Waiting for close lock at 1730983324446Running coprocessor pre-close hooks at 1730983324446Disabling compacts and flushes for region at 1730983324446Disabling writes for close at 1730983324446Obtaining lock to block concurrent updates at 1730983324446Preparing flush snapshotting stores in 90c0d1c8af782719d9f8d037e72c2a12 at 1730983324446Finished memstore snapshotting TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1730983324447 (+1 ms)Flushing stores of TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12. at 1730983324447Flushing 90c0d1c8af782719d9f8d037e72c2a12/A: creating writer at 1730983324447Flushing mob file 90c0d1c8af782719d9f8d037e72c2a12/A: appending metadata at 1730983324454 (+7 ms)Flushing mob file 90c0d1c8af782719d9f8d037e72c2a12/A: closing flushed file at 1730983324454Flushing 90c0d1c8af782719d9f8d037e72c2a12/A: appending metadata at 1730983324865 (+411 ms)Flushing 90c0d1c8af782719d9f8d037e72c2a12/A: closing flushed file at 1730983324865Flushing 90c0d1c8af782719d9f8d037e72c2a12/B: creating writer at 1730983325275 (+410 ms)Flushing 90c0d1c8af782719d9f8d037e72c2a12/B: appending metadata at 1730983325277 (+2 ms)Flushing 90c0d1c8af782719d9f8d037e72c2a12/B: closing flushed file at 1730983325277Flushing 90c0d1c8af782719d9f8d037e72c2a12/C: creating writer at 1730983325686 (+409 ms)Flushing 90c0d1c8af782719d9f8d037e72c2a12/C: appending metadata at 1730983325690 (+4 ms)Flushing 90c0d1c8af782719d9f8d037e72c2a12/C: closing flushed file at 1730983325690Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@39b829ca: reopening flushed file at 1730983326099 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@72b4fc: reopening flushed file at 1730983326104 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6886244d: reopening flushed file at 1730983326108 (+4 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 90c0d1c8af782719d9f8d037e72c2a12 in 1668ms, sequenceid=459, compaction requested=false at 1730983326114 (+6 ms)Writing region close event to WAL at 1730983326232 (+118 ms)Running coprocessor post-close hooks at 1730983326236 (+4 ms)Closed at 1730983326237 (+1 ms) 2024-11-07T12:42:06,239 INFO [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(157): Closed 90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:42:06,239 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=90c0d1c8af782719d9f8d037e72c2a12, regionState=CLOSED 2024-11-07T12:42:06,241 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=58, state=RUNNABLE, hasLock=false; CloseRegionProcedure 90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 because future has completed 2024-11-07T12:42:06,243 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-07T12:42:06,243 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; CloseRegionProcedure 90c0d1c8af782719d9f8d037e72c2a12, server=db9ad1cb6cf9,34293,1730983260217 in 1.9480 sec 2024-11-07T12:42:06,245 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-07T12:42:06,245 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=90c0d1c8af782719d9f8d037e72c2a12, UNASSIGN in 1.9520 sec 2024-11-07T12:42:06,247 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-07T12:42:06,247 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9570 sec 2024-11-07T12:42:06,249 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730983326249"}]},"ts":"1730983326249"} 2024-11-07T12:42:06,250 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-07T12:42:06,250 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-07T12:42:06,252 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9660 sec 2024-11-07T12:42:06,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-07T12:42:06,432 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:42:06,433 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-07T12:42:06,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:42:06,435 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=60, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:42:06,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-07T12:42:06,436 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=60, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:42:06,438 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:42:06,441 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A, FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B, FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C, FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/recovered.edits] 2024-11-07T12:42:06,443 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/a9a626c1b0d942b2a6191b0745107b55 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/a9a626c1b0d942b2a6191b0745107b55 2024-11-07T12:42:06,445 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/ae8bfe27110f4dd8b2c78c0432d8c19c to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/A/ae8bfe27110f4dd8b2c78c0432d8c19c 2024-11-07T12:42:06,447 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/09209ff30ed64e03b684aeb35162d0cf to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/09209ff30ed64e03b684aeb35162d0cf 2024-11-07T12:42:06,448 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/ba2a13013e904eaf9025f1a9e38d362a to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/B/ba2a13013e904eaf9025f1a9e38d362a 2024-11-07T12:42:06,450 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/ef89e9c3a5db4368b1b9c567de8d25b2 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/ef89e9c3a5db4368b1b9c567de8d25b2 2024-11-07T12:42:06,451 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/f0ef499135c04c13803cfc87ff77c196 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/C/f0ef499135c04c13803cfc87ff77c196 2024-11-07T12:42:06,454 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/recovered.edits/462.seqid to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12/recovered.edits/462.seqid 2024-11-07T12:42:06,455 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:42:06,455 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-07T12:42:06,455 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-07T12:42:06,456 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-07T12:42:06,461 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411070c9e13ad68724ad8a18c6f4b7f2e9078_90c0d1c8af782719d9f8d037e72c2a12 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411070c9e13ad68724ad8a18c6f4b7f2e9078_90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:42:06,463 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110714ee5edac1414cc3af3e6baed8bb846f_90c0d1c8af782719d9f8d037e72c2a12 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110714ee5edac1414cc3af3e6baed8bb846f_90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:42:06,467 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110718ec6594ef684861afdc6577a0f829e3_90c0d1c8af782719d9f8d037e72c2a12 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110718ec6594ef684861afdc6577a0f829e3_90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:42:06,471 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110733404f76130445139df5da0c20d14d1e_90c0d1c8af782719d9f8d037e72c2a12 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110733404f76130445139df5da0c20d14d1e_90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:42:06,480 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107382620a1e2f0459cad2f52342acb46ac_90c0d1c8af782719d9f8d037e72c2a12 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107382620a1e2f0459cad2f52342acb46ac_90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:42:06,481 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110739cb4a3e4ffb4fbfaace6ea97d7311ae_90c0d1c8af782719d9f8d037e72c2a12 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110739cb4a3e4ffb4fbfaace6ea97d7311ae_90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:42:06,482 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411073b9d9479a10e4761b2801f450a108b2c_90c0d1c8af782719d9f8d037e72c2a12 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411073b9d9479a10e4761b2801f450a108b2c_90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:42:06,483 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411073f30dd36805c4c249565739bd21cc858_90c0d1c8af782719d9f8d037e72c2a12 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411073f30dd36805c4c249565739bd21cc858_90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:42:06,485 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411074bcd381d40884f7fb44bf3bac7c7d4f5_90c0d1c8af782719d9f8d037e72c2a12 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411074bcd381d40884f7fb44bf3bac7c7d4f5_90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:42:06,486 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107508e020ff1644ac6b7e1270d588a0614_90c0d1c8af782719d9f8d037e72c2a12 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107508e020ff1644ac6b7e1270d588a0614_90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:42:06,487 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107755bb953d4f748138c3beef211638ef7_90c0d1c8af782719d9f8d037e72c2a12 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107755bb953d4f748138c3beef211638ef7_90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:42:06,488 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107778b5e8a3d974b19a5459b7379be1339_90c0d1c8af782719d9f8d037e72c2a12 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107778b5e8a3d974b19a5459b7379be1339_90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:42:06,490 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110778f1add6eb834223a6ccc2b1b0d918fd_90c0d1c8af782719d9f8d037e72c2a12 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110778f1add6eb834223a6ccc2b1b0d918fd_90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:42:06,491 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107879f6a3833264cb08c1cf32d6f84f6ba_90c0d1c8af782719d9f8d037e72c2a12 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107879f6a3833264cb08c1cf32d6f84f6ba_90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:42:06,492 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110794c3d5ef8d8c498eaa4b94fdc8e96535_90c0d1c8af782719d9f8d037e72c2a12 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110794c3d5ef8d8c498eaa4b94fdc8e96535_90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:42:06,493 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107a6cb6a6f326d41a6a01e48905a25ae8c_90c0d1c8af782719d9f8d037e72c2a12 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107a6cb6a6f326d41a6a01e48905a25ae8c_90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:42:06,494 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107ba38f17ddd664ffd815eace5964e408e_90c0d1c8af782719d9f8d037e72c2a12 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107ba38f17ddd664ffd815eace5964e408e_90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:42:06,495 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107bf81e9544c7d431bbb7ce9415d715358_90c0d1c8af782719d9f8d037e72c2a12 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107bf81e9544c7d431bbb7ce9415d715358_90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:42:06,497 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107d0ecbadfdb5a4d80b85d426221ef9284_90c0d1c8af782719d9f8d037e72c2a12 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107d0ecbadfdb5a4d80b85d426221ef9284_90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:42:06,498 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107d44d9f700ec04d4db39c065fbc5d6969_90c0d1c8af782719d9f8d037e72c2a12 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107d44d9f700ec04d4db39c065fbc5d6969_90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:42:06,499 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107e607456ef5894f76b4d6c660fc411a02_90c0d1c8af782719d9f8d037e72c2a12 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107e607456ef5894f76b4d6c660fc411a02_90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:42:06,500 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107ed27035721094552ad73611ff0ec234f_90c0d1c8af782719d9f8d037e72c2a12 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107ed27035721094552ad73611ff0ec234f_90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:42:06,502 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107f604959604024fdb85ef7e58f73aeeca_90c0d1c8af782719d9f8d037e72c2a12 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107f604959604024fdb85ef7e58f73aeeca_90c0d1c8af782719d9f8d037e72c2a12 2024-11-07T12:42:06,502 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-07T12:42:06,505 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=60, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:42:06,508 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-07T12:42:06,510 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-07T12:42:06,512 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=60, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:42:06,512 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-07T12:42:06,512 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1730983326512"}]},"ts":"9223372036854775807"} 2024-11-07T12:42:06,514 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-07T12:42:06,514 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 90c0d1c8af782719d9f8d037e72c2a12, NAME => 'TestAcidGuarantees,,1730983296817.90c0d1c8af782719d9f8d037e72c2a12.', STARTKEY => '', ENDKEY => ''}] 2024-11-07T12:42:06,515 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-07T12:42:06,515 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1730983326515"}]},"ts":"9223372036854775807"} 2024-11-07T12:42:06,517 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-07T12:42:06,517 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=60, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:42:06,519 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 84 msec 2024-11-07T12:42:06,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-07T12:42:06,542 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-07T12:42:06,542 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:42:06,553 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=251 (was 245) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ba7e58d5-de0c-7392-0264-ead072f99c84/cluster_73ed722e-8252-77e5-7683-f63666bd3476/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-269572622_22 at /127.0.0.1:34356 [Waiting for operation #1055] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-269572622_22 at /127.0.0.1:34416 [Waiting for operation #1056] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ba7e58d5-de0c-7392-0264-ead072f99c84/cluster_73ed722e-8252-77e5-7683-f63666bd3476/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1483776571_22 at /127.0.0.1:34434 [Waiting for operation #769] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-269572622_22 at /127.0.0.1:42604 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1483776571_22 at /127.0.0.1:54914 [Waiting for operation #1076] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=463 (was 449) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1224 (was 1552), ProcessCount=11 (was 11), AvailableMemoryMB=2622 (was 3057) 2024-11-07T12:42:06,571 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=251, OpenFileDescriptor=463, MaxFileDescriptor=1048576, SystemLoadAverage=1224, ProcessCount=11, AvailableMemoryMB=2622 2024-11-07T12:42:06,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-07T12:42:06,573 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T12:42:06,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] procedure2.ProcedureExecutor(1139): Stored pid=61, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-07T12:42:06,579 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T12:42:06,579 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:06,579 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 61 2024-11-07T12:42:06,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-07T12:42:06,581 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T12:42:06,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742067_1243 (size=1061) 2024-11-07T12:42:06,604 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4 2024-11-07T12:42:06,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742068_1244 (size=53) 2024-11-07T12:42:06,623 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:42:06,623 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 8dc13886c19c8110f075e43282960321, disabling compactions & flushes 2024-11-07T12:42:06,623 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:06,623 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:06,623 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. after waiting 0 ms 2024-11-07T12:42:06,623 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:06,623 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:06,623 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8dc13886c19c8110f075e43282960321: Waiting for close lock at 1730983326623Disabling compacts and flushes for region at 1730983326623Disabling writes for close at 1730983326623Writing region close event to WAL at 1730983326623Closed at 1730983326623 2024-11-07T12:42:06,624 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T12:42:06,625 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1730983326625"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730983326625"}]},"ts":"1730983326625"} 2024-11-07T12:42:06,627 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-07T12:42:06,628 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T12:42:06,628 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730983326628"}]},"ts":"1730983326628"} 2024-11-07T12:42:06,630 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-07T12:42:06,630 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8dc13886c19c8110f075e43282960321, ASSIGN}] 2024-11-07T12:42:06,631 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8dc13886c19c8110f075e43282960321, ASSIGN 2024-11-07T12:42:06,632 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8dc13886c19c8110f075e43282960321, ASSIGN; state=OFFLINE, location=db9ad1cb6cf9,34293,1730983260217; forceNewPlan=false, retain=false 2024-11-07T12:42:06,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-07T12:42:06,783 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=8dc13886c19c8110f075e43282960321, regionState=OPENING, regionLocation=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:06,785 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8dc13886c19c8110f075e43282960321, ASSIGN because future has completed 2024-11-07T12:42:06,785 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217}] 2024-11-07T12:42:06,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-07T12:42:06,940 INFO [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:06,940 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7752): Opening region: {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} 2024-11-07T12:42:06,940 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 8dc13886c19c8110f075e43282960321 2024-11-07T12:42:06,940 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:42:06,940 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7794): checking encryption for 8dc13886c19c8110f075e43282960321 2024-11-07T12:42:06,940 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7797): checking classloading for 8dc13886c19c8110f075e43282960321 2024-11-07T12:42:06,942 INFO [StoreOpener-8dc13886c19c8110f075e43282960321-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 8dc13886c19c8110f075e43282960321 2024-11-07T12:42:06,943 INFO [StoreOpener-8dc13886c19c8110f075e43282960321-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-07T12:42:06,943 INFO [StoreOpener-8dc13886c19c8110f075e43282960321-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8dc13886c19c8110f075e43282960321 columnFamilyName A 2024-11-07T12:42:06,943 DEBUG [StoreOpener-8dc13886c19c8110f075e43282960321-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:06,944 INFO [StoreOpener-8dc13886c19c8110f075e43282960321-1 {}] regionserver.HStore(327): Store=8dc13886c19c8110f075e43282960321/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:42:06,944 INFO [StoreOpener-8dc13886c19c8110f075e43282960321-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 8dc13886c19c8110f075e43282960321 2024-11-07T12:42:06,945 INFO [StoreOpener-8dc13886c19c8110f075e43282960321-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-07T12:42:06,945 INFO [StoreOpener-8dc13886c19c8110f075e43282960321-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8dc13886c19c8110f075e43282960321 columnFamilyName B 2024-11-07T12:42:06,945 DEBUG [StoreOpener-8dc13886c19c8110f075e43282960321-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:06,945 INFO [StoreOpener-8dc13886c19c8110f075e43282960321-1 {}] regionserver.HStore(327): Store=8dc13886c19c8110f075e43282960321/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:42:06,945 INFO [StoreOpener-8dc13886c19c8110f075e43282960321-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 8dc13886c19c8110f075e43282960321 2024-11-07T12:42:06,946 INFO [StoreOpener-8dc13886c19c8110f075e43282960321-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-07T12:42:06,947 INFO [StoreOpener-8dc13886c19c8110f075e43282960321-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8dc13886c19c8110f075e43282960321 columnFamilyName C 2024-11-07T12:42:06,947 DEBUG [StoreOpener-8dc13886c19c8110f075e43282960321-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:06,947 INFO [StoreOpener-8dc13886c19c8110f075e43282960321-1 {}] regionserver.HStore(327): Store=8dc13886c19c8110f075e43282960321/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:42:06,947 INFO [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:06,947 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1038): replaying wal for 8dc13886c19c8110f075e43282960321 2024-11-07T12:42:06,948 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321 2024-11-07T12:42:06,948 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321 2024-11-07T12:42:06,949 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1048): stopping wal replay for 8dc13886c19c8110f075e43282960321 2024-11-07T12:42:06,949 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1060): Cleaning up temporary data for 8dc13886c19c8110f075e43282960321 2024-11-07T12:42:06,950 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-07T12:42:06,951 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1093): writing seq id for 8dc13886c19c8110f075e43282960321 2024-11-07T12:42:06,953 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T12:42:06,954 INFO [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1114): Opened 8dc13886c19c8110f075e43282960321; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59942916, jitterRate=-0.10678094625473022}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-07T12:42:06,954 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8dc13886c19c8110f075e43282960321 2024-11-07T12:42:06,954 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1006): Region open journal for 8dc13886c19c8110f075e43282960321: Running coprocessor pre-open hook at 1730983326940Writing region info on filesystem at 1730983326940Initializing all the Stores at 1730983326941 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730983326941Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730983326941Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730983326941Cleaning up temporary data from old regions at 1730983326950 (+9 ms)Running coprocessor post-open hooks at 1730983326954 (+4 ms)Region opened successfully at 1730983326954 2024-11-07T12:42:06,955 INFO [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., pid=63, masterSystemTime=1730983326937 2024-11-07T12:42:06,957 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:06,957 INFO [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:06,957 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=8dc13886c19c8110f075e43282960321, regionState=OPEN, openSeqNum=2, regionLocation=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:06,959 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=63, ppid=62, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 because future has completed 2024-11-07T12:42:06,961 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-07T12:42:06,961 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; OpenRegionProcedure 8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 in 175 msec 2024-11-07T12:42:06,963 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-11-07T12:42:06,963 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8dc13886c19c8110f075e43282960321, ASSIGN in 331 msec 2024-11-07T12:42:06,964 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T12:42:06,964 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730983326964"}]},"ts":"1730983326964"} 2024-11-07T12:42:06,966 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-07T12:42:06,966 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T12:42:06,968 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 393 msec 2024-11-07T12:42:07,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-07T12:42:07,212 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:42:07,213 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d8da64d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:42:07,213 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request db9ad1cb6cf9,38619,-1 for getting cluster id 2024-11-07T12:42:07,213 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:42:07,214 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e542096-4b0e-4b01-9952-662501fc0f65' 2024-11-07T12:42:07,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:42:07,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e542096-4b0e-4b01-9952-662501fc0f65" 2024-11-07T12:42:07,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b6313f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:42:07,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [db9ad1cb6cf9,38619,-1] 2024-11-07T12:42:07,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:42:07,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:42:07,216 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54438, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:42:07,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72558a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:42:07,217 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:42:07,217 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=-1] 2024-11-07T12:42:07,218 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:42:07,219 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41974, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:42:07,219 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-07T12:42:07,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is db9ad1cb6cf9,38619,1730983259119 2024-11-07T12:42:07,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@72f7ee7d 2024-11-07T12:42:07,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-07T12:42:07,221 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54448, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-07T12:42:07,222 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@554689fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:42:07,222 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request db9ad1cb6cf9,38619,-1 for getting cluster id 2024-11-07T12:42:07,222 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:42:07,223 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e542096-4b0e-4b01-9952-662501fc0f65' 2024-11-07T12:42:07,223 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:42:07,223 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e542096-4b0e-4b01-9952-662501fc0f65" 2024-11-07T12:42:07,223 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49adc487, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:42:07,223 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [db9ad1cb6cf9,38619,-1] 2024-11-07T12:42:07,223 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:42:07,224 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:42:07,224 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54466, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:42:07,225 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8c98841, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:42:07,225 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72325ab8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:42:07,225 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request db9ad1cb6cf9,38619,-1 for getting cluster id 2024-11-07T12:42:07,225 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:42:07,226 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e542096-4b0e-4b01-9952-662501fc0f65' 2024-11-07T12:42:07,226 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:42:07,226 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e542096-4b0e-4b01-9952-662501fc0f65" 2024-11-07T12:42:07,226 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@622c3e25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:42:07,226 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [db9ad1cb6cf9,38619,-1] 2024-11-07T12:42:07,226 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:42:07,226 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:42:07,227 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54482, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:42:07,228 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31836dcf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:42:07,228 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6686bc48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:42:07,228 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request db9ad1cb6cf9,38619,-1 for getting cluster id 2024-11-07T12:42:07,228 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:42:07,229 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e542096-4b0e-4b01-9952-662501fc0f65' 2024-11-07T12:42:07,229 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:42:07,229 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e542096-4b0e-4b01-9952-662501fc0f65" 2024-11-07T12:42:07,229 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3699da99, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:42:07,229 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [db9ad1cb6cf9,38619,-1] 2024-11-07T12:42:07,229 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:42:07,229 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:42:07,230 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54494, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:42:07,230 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75db2b02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:42:07,231 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d70004e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:42:07,231 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request db9ad1cb6cf9,38619,-1 for getting cluster id 2024-11-07T12:42:07,231 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:42:07,232 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e542096-4b0e-4b01-9952-662501fc0f65' 2024-11-07T12:42:07,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:42:07,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e542096-4b0e-4b01-9952-662501fc0f65" 2024-11-07T12:42:07,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f197f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:42:07,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [db9ad1cb6cf9,38619,-1] 2024-11-07T12:42:07,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:42:07,233 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:42:07,233 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54516, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:42:07,234 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58fc94e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:42:07,234 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19903550, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:42:07,234 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request db9ad1cb6cf9,38619,-1 for getting cluster id 2024-11-07T12:42:07,234 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:42:07,235 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e542096-4b0e-4b01-9952-662501fc0f65' 2024-11-07T12:42:07,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:42:07,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e542096-4b0e-4b01-9952-662501fc0f65" 2024-11-07T12:42:07,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6584c217, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:42:07,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [db9ad1cb6cf9,38619,-1] 2024-11-07T12:42:07,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:42:07,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:42:07,236 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54546, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:42:07,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1acef2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:42:07,237 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b9635b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:42:07,237 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request db9ad1cb6cf9,38619,-1 for getting cluster id 2024-11-07T12:42:07,237 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:42:07,238 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e542096-4b0e-4b01-9952-662501fc0f65' 2024-11-07T12:42:07,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:42:07,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e542096-4b0e-4b01-9952-662501fc0f65" 2024-11-07T12:42:07,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@580b3fbe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:42:07,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [db9ad1cb6cf9,38619,-1] 2024-11-07T12:42:07,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:42:07,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:42:07,239 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54558, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:42:07,239 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34eccea9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:42:07,240 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2611025, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:42:07,240 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request db9ad1cb6cf9,38619,-1 for getting cluster id 2024-11-07T12:42:07,240 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:42:07,241 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e542096-4b0e-4b01-9952-662501fc0f65' 2024-11-07T12:42:07,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:42:07,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e542096-4b0e-4b01-9952-662501fc0f65" 2024-11-07T12:42:07,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@213b3701, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:42:07,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [db9ad1cb6cf9,38619,-1] 2024-11-07T12:42:07,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:42:07,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:42:07,242 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54570, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:42:07,242 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@639b3112, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:42:07,243 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8a8ae50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:42:07,243 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request db9ad1cb6cf9,38619,-1 for getting cluster id 2024-11-07T12:42:07,243 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:42:07,244 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e542096-4b0e-4b01-9952-662501fc0f65' 2024-11-07T12:42:07,244 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:42:07,244 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e542096-4b0e-4b01-9952-662501fc0f65" 2024-11-07T12:42:07,244 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d5d4291, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:42:07,244 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [db9ad1cb6cf9,38619,-1] 2024-11-07T12:42:07,244 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:42:07,244 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:42:07,245 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54590, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:42:07,245 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f212f53, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:42:07,246 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@471771d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:42:07,246 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request db9ad1cb6cf9,38619,-1 for getting cluster id 2024-11-07T12:42:07,246 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:42:07,247 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e542096-4b0e-4b01-9952-662501fc0f65' 2024-11-07T12:42:07,247 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:42:07,247 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e542096-4b0e-4b01-9952-662501fc0f65" 2024-11-07T12:42:07,247 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@387d2fa8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:42:07,247 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [db9ad1cb6cf9,38619,-1] 2024-11-07T12:42:07,247 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:42:07,248 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:42:07,248 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54614, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:42:07,249 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66eb92ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:42:07,249 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40916ac2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:42:07,249 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request db9ad1cb6cf9,38619,-1 for getting cluster id 2024-11-07T12:42:07,249 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:42:07,250 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e542096-4b0e-4b01-9952-662501fc0f65' 2024-11-07T12:42:07,250 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:42:07,250 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e542096-4b0e-4b01-9952-662501fc0f65" 2024-11-07T12:42:07,250 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@515b3bd2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:42:07,250 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [db9ad1cb6cf9,38619,-1] 2024-11-07T12:42:07,250 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:42:07,250 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:42:07,251 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54630, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:42:07,251 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b8f4317, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:42:07,252 DEBUG [Thread-1154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:42:07,252 DEBUG [Thread-1143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:42:07,252 DEBUG [Thread-1139 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:42:07,252 DEBUG [Thread-1148 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:42:07,252 DEBUG [Thread-1152 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:42:07,253 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=-1] 2024-11-07T12:42:07,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=-1] 2024-11-07T12:42:07,253 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=-1] 2024-11-07T12:42:07,253 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=-1] 2024-11-07T12:42:07,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:42:07,253 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:42:07,253 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=-1] 2024-11-07T12:42:07,253 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:42:07,253 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:42:07,253 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:42:07,254 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41990, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:42:07,254 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42010, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:42:07,255 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42006, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:42:07,255 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41976, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:42:07,255 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42026, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:42:07,256 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2] 2024-11-07T12:42:07,256 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2] 2024-11-07T12:42:07,257 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:42:07,257 DEBUG [Thread-1145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:42:07,258 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2] 2024-11-07T12:42:07,258 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2] 2024-11-07T12:42:07,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2] 2024-11-07T12:42:07,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=-1] 2024-11-07T12:42:07,260 DEBUG [Thread-1156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:42:07,260 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:42:07,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-07T12:42:07,261 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=-1] 2024-11-07T12:42:07,261 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:42:07,261 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:42:07,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-07T12:42:07,262 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42034, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:42:07,262 DEBUG [Thread-1150 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:42:07,263 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42044, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:42:07,263 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:42:07,263 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:42:07,264 DEBUG [Thread-1150 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=-1] 2024-11-07T12:42:07,264 DEBUG [Thread-1150 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:42:07,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2] 2024-11-07T12:42:07,265 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42052, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:42:07,265 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2] 2024-11-07T12:42:07,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2] 2024-11-07T12:42:07,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 8dc13886c19c8110f075e43282960321 2024-11-07T12:42:07,270 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8dc13886c19c8110f075e43282960321 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-07T12:42:07,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=A 2024-11-07T12:42:07,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:07,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=B 2024-11-07T12:42:07,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:07,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=C 2024-11-07T12:42:07,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:07,277 DEBUG [Thread-1137 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:42:07,277 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=-1] 2024-11-07T12:42:07,277 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:42:07,278 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42054, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:42:07,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2] 2024-11-07T12:42:07,293 DEBUG [Thread-1141 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:42:07,321 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=-1] 2024-11-07T12:42:07,322 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:42:07,324 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42070, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:42:07,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2] 2024-11-07T12:42:07,342 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/04a388aed75a40b0ad01201ef25b1bab is 50, key is test_row_0/A:col10/1730983327270/Put/seqid=0 2024-11-07T12:42:07,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:07,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41976 deadline: 1730983387343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:07,356 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:07,356 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:07,356 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:07,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:07,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42054 deadline: 1730983387344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:07,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:07,357 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:07,357 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:07,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42034 deadline: 1730983387344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:07,357 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:07,357 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:07,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:07,357 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:07,357 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:07,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 2 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42070 deadline: 1730983387345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:07,358 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:07,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:07,358 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:07,358 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:07,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42026 deadline: 1730983387346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:07,359 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:07,359 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:07,359 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:07,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-07T12:42:07,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742069_1245 (size=12001) 2024-11-07T12:42:07,403 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/04a388aed75a40b0ad01201ef25b1bab 2024-11-07T12:42:07,418 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-07T12:42:07,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:07,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. as already flushing 2024-11-07T12:42:07,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:07,419 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:07,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:07,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:07,442 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/5c968edfd75f491db5fedc7a63589a8e is 50, key is test_row_0/B:col10/1730983327270/Put/seqid=0 2024-11-07T12:42:07,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:07,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42054 deadline: 1730983387462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:07,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:07,467 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:07,467 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:07,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42070 deadline: 1730983387462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:07,467 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:07,468 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:07,468 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:07,469 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:07,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:07,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42026 deadline: 1730983387462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:07,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:07,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:07,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:07,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:07,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41976 deadline: 1730983387462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:07,470 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:07,470 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:07,470 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:07,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:07,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42034 deadline: 1730983387463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:07,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:07,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:07,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:07,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742070_1246 (size=12001) 2024-11-07T12:42:07,574 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-07T12:42:07,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:07,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. as already flushing 2024-11-07T12:42:07,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:07,574 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:07,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:07,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:07,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-07T12:42:07,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:07,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42054 deadline: 1730983387672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:07,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:07,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42070 deadline: 1730983387672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:07,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:07,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:07,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:07,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:07,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:07,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:07,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:07,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42026 deadline: 1730983387681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:07,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:07,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:07,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:07,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:07,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41976 deadline: 1730983387681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:07,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:07,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42034 deadline: 1730983387682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:07,685 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:07,685 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:07,685 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:07,685 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:07,685 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:07,685 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:07,733 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-07T12:42:07,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:07,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. as already flushing 2024-11-07T12:42:07,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:07,733 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:07,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:07,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:07,889 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-07T12:42:07,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:07,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. as already flushing 2024-11-07T12:42:07,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:07,889 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:07,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:07,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:07,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-07T12:42:07,896 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/5c968edfd75f491db5fedc7a63589a8e 2024-11-07T12:42:07,944 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/07f8545df3d647dfb97011bf2da37603 is 50, key is test_row_0/C:col10/1730983327270/Put/seqid=0 2024-11-07T12:42:07,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:07,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42054 deadline: 1730983387985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:07,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:07,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:07,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:07,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:07,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42070 deadline: 1730983387985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:07,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742071_1247 (size=12001) 2024-11-07T12:42:07,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:07,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:07,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:07,993 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/07f8545df3d647dfb97011bf2da37603 2024-11-07T12:42:07,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:07,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42034 deadline: 1730983387992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:08,000 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:08,000 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:08,000 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:08,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:08,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42026 deadline: 1730983387992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:08,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:08,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:08,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:08,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:08,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41976 deadline: 1730983387992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:08,002 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:08,002 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:08,002 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:08,016 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/04a388aed75a40b0ad01201ef25b1bab as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/04a388aed75a40b0ad01201ef25b1bab 2024-11-07T12:42:08,037 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/04a388aed75a40b0ad01201ef25b1bab, entries=150, sequenceid=13, filesize=11.7 K 2024-11-07T12:42:08,042 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/5c968edfd75f491db5fedc7a63589a8e as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/5c968edfd75f491db5fedc7a63589a8e 2024-11-07T12:42:08,043 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-07T12:42:08,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:08,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. as already flushing 2024-11-07T12:42:08,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:08,043 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:08,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:08,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:08,062 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/5c968edfd75f491db5fedc7a63589a8e, entries=150, sequenceid=13, filesize=11.7 K 2024-11-07T12:42:08,063 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/07f8545df3d647dfb97011bf2da37603 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/07f8545df3d647dfb97011bf2da37603 2024-11-07T12:42:08,069 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/07f8545df3d647dfb97011bf2da37603, entries=150, sequenceid=13, filesize=11.7 K 2024-11-07T12:42:08,070 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 8dc13886c19c8110f075e43282960321 in 800ms, sequenceid=13, compaction requested=false 2024-11-07T12:42:08,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8dc13886c19c8110f075e43282960321: 2024-11-07T12:42:08,196 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-07T12:42:08,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:08,197 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 8dc13886c19c8110f075e43282960321 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-07T12:42:08,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=A 2024-11-07T12:42:08,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:08,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=B 2024-11-07T12:42:08,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:08,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=C 2024-11-07T12:42:08,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:08,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/e6041cbd6b634c7083d035ea8d8da73e is 50, key is test_row_0/A:col10/1730983327343/Put/seqid=0 2024-11-07T12:42:08,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742072_1248 (size=12001) 2024-11-07T12:42:08,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-07T12:42:08,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 8dc13886c19c8110f075e43282960321 2024-11-07T12:42:08,504 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. as already flushing 2024-11-07T12:42:08,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:08,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41976 deadline: 1730983388517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:08,525 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:08,525 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:08,524 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:08,525 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:08,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42034 deadline: 1730983388519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:08,525 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:08,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42026 deadline: 1730983388520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:08,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:08,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42054 deadline: 1730983388523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:08,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:08,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42070 deadline: 1730983388525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:08,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:08,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:08,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:08,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:08,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:08,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:08,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:08,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:08,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:08,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:08,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:08,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:08,625 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/e6041cbd6b634c7083d035ea8d8da73e 2024-11-07T12:42:08,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:08,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42026 deadline: 1730983388632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:08,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:08,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:08,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:08,634 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:08,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42034 deadline: 1730983388632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:08,634 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:08,634 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:08,634 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:08,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:08,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42054 deadline: 1730983388632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:08,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:08,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:08,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:08,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:08,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42070 deadline: 1730983388633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:08,637 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:08,637 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:08,637 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:08,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:08,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41976 deadline: 1730983388633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:08,638 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:08,638 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:08,638 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:08,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/254b0d38fd9a4fcf80b0a0e3fcc72b82 is 50, key is test_row_0/B:col10/1730983327343/Put/seqid=0 2024-11-07T12:42:08,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742073_1249 (size=12001) 2024-11-07T12:42:08,680 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/254b0d38fd9a4fcf80b0a0e3fcc72b82 2024-11-07T12:42:08,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/883d4efb87a34329bc3324daa3403b1a is 50, key is test_row_0/C:col10/1730983327343/Put/seqid=0 2024-11-07T12:42:08,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742074_1250 (size=12001) 2024-11-07T12:42:08,737 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/883d4efb87a34329bc3324daa3403b1a 2024-11-07T12:42:08,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/e6041cbd6b634c7083d035ea8d8da73e as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/e6041cbd6b634c7083d035ea8d8da73e 2024-11-07T12:42:08,753 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/e6041cbd6b634c7083d035ea8d8da73e, entries=150, sequenceid=33, filesize=11.7 K 2024-11-07T12:42:08,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/254b0d38fd9a4fcf80b0a0e3fcc72b82 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/254b0d38fd9a4fcf80b0a0e3fcc72b82 2024-11-07T12:42:08,761 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/254b0d38fd9a4fcf80b0a0e3fcc72b82, entries=150, sequenceid=33, filesize=11.7 K 2024-11-07T12:42:08,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/883d4efb87a34329bc3324daa3403b1a as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/883d4efb87a34329bc3324daa3403b1a 2024-11-07T12:42:08,781 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/883d4efb87a34329bc3324daa3403b1a, entries=150, sequenceid=33, filesize=11.7 K 2024-11-07T12:42:08,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.StoreScanner(997): StoreScanner already has the close lock. There is no need to updateReaders 2024-11-07T12:42:08,784 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 8dc13886c19c8110f075e43282960321 in 587ms, sequenceid=33, compaction requested=false 2024-11-07T12:42:08,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 8dc13886c19c8110f075e43282960321: 2024-11-07T12:42:08,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:08,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-07T12:42:08,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-07T12:42:08,803 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-07T12:42:08,803 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5250 sec 2024-11-07T12:42:08,805 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 1.5460 sec 2024-11-07T12:42:08,844 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8dc13886c19c8110f075e43282960321 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-07T12:42:08,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 8dc13886c19c8110f075e43282960321 2024-11-07T12:42:08,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=A 2024-11-07T12:42:08,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:08,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=B 2024-11-07T12:42:08,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:08,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=C 2024-11-07T12:42:08,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:08,859 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/240af32ce2724ad996f2a04c71cb3710 is 50, key is test_row_0/A:col10/1730983328521/Put/seqid=0 2024-11-07T12:42:08,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:08,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42070 deadline: 1730983388877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:08,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742075_1251 (size=12001) 2024-11-07T12:42:08,885 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:08,885 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:08,885 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:08,886 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/240af32ce2724ad996f2a04c71cb3710 2024-11-07T12:42:08,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:08,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42034 deadline: 1730983388882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:08,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:08,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:08,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:08,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:08,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41976 deadline: 1730983388883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:08,895 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:08,895 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:08,895 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:08,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:08,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42054 deadline: 1730983388885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:08,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:08,895 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:08,896 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:08,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42026 deadline: 1730983388886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:08,896 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:08,896 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:08,896 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:08,896 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:08,898 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/68e78311dd5540afb6c30605edae96be is 50, key is test_row_0/B:col10/1730983328521/Put/seqid=0 2024-11-07T12:42:08,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742076_1252 (size=12001) 2024-11-07T12:42:09,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:09,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42070 deadline: 1730983388992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:09,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:09,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42034 deadline: 1730983389002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:09,004 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:09,004 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:09,004 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:09,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:09,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42054 deadline: 1730983389002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:09,008 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:09,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:09,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:09,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:09,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41976 deadline: 1730983389002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:09,010 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:09,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:09,010 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:09,010 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:09,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42026 deadline: 1730983389002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:09,010 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:09,010 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:09,010 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:09,012 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:09,012 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:09,012 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:09,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:09,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42034 deadline: 1730983389212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:09,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:09,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42070 deadline: 1730983389212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:09,214 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:09,214 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:09,214 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:09,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:09,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41976 deadline: 1730983389212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:09,215 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:09,215 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:09,215 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:09,215 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:09,215 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:09,215 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:09,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:09,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42054 deadline: 1730983389221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:09,226 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:09,226 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:09,226 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:09,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:09,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42026 deadline: 1730983389221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:09,226 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:09,226 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:09,226 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:09,335 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/68e78311dd5540afb6c30605edae96be 2024-11-07T12:42:09,364 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/06c86adf7f6342a2b61c285712d237d0 is 50, key is test_row_0/C:col10/1730983328521/Put/seqid=0 2024-11-07T12:42:09,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742077_1253 (size=12001) 2024-11-07T12:42:09,401 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/06c86adf7f6342a2b61c285712d237d0 2024-11-07T12:42:09,408 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/240af32ce2724ad996f2a04c71cb3710 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/240af32ce2724ad996f2a04c71cb3710 2024-11-07T12:42:09,416 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/240af32ce2724ad996f2a04c71cb3710, entries=150, sequenceid=48, filesize=11.7 K 2024-11-07T12:42:09,417 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/68e78311dd5540afb6c30605edae96be as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/68e78311dd5540afb6c30605edae96be 2024-11-07T12:42:09,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-07T12:42:09,419 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:42:09,419 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:42:09,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-07T12:42:09,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-07T12:42:09,421 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:42:09,422 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:42:09,422 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:42:09,423 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/68e78311dd5540afb6c30605edae96be, entries=150, sequenceid=48, filesize=11.7 K 2024-11-07T12:42:09,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/06c86adf7f6342a2b61c285712d237d0 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/06c86adf7f6342a2b61c285712d237d0 2024-11-07T12:42:09,432 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/06c86adf7f6342a2b61c285712d237d0, entries=150, sequenceid=48, filesize=11.7 K 2024-11-07T12:42:09,435 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 8dc13886c19c8110f075e43282960321 in 591ms, sequenceid=48, compaction requested=true 2024-11-07T12:42:09,435 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8dc13886c19c8110f075e43282960321: 2024-11-07T12:42:09,435 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:42:09,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8dc13886c19c8110f075e43282960321:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:42:09,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:42:09,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8dc13886c19c8110f075e43282960321:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:42:09,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:42:09,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8dc13886c19c8110f075e43282960321:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:42:09,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:42:09,435 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:42:09,437 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:42:09,437 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): 8dc13886c19c8110f075e43282960321/A is initiating minor compaction (all files) 2024-11-07T12:42:09,437 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8dc13886c19c8110f075e43282960321/A in TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:09,437 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/04a388aed75a40b0ad01201ef25b1bab, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/e6041cbd6b634c7083d035ea8d8da73e, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/240af32ce2724ad996f2a04c71cb3710] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp, totalSize=35.2 K 2024-11-07T12:42:09,437 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:42:09,437 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 8dc13886c19c8110f075e43282960321/B is initiating minor compaction (all files) 2024-11-07T12:42:09,437 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8dc13886c19c8110f075e43282960321/B in TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:09,438 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/5c968edfd75f491db5fedc7a63589a8e, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/254b0d38fd9a4fcf80b0a0e3fcc72b82, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/68e78311dd5540afb6c30605edae96be] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp, totalSize=35.2 K 2024-11-07T12:42:09,438 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04a388aed75a40b0ad01201ef25b1bab, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1730983327262 2024-11-07T12:42:09,438 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c968edfd75f491db5fedc7a63589a8e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1730983327262 2024-11-07T12:42:09,439 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6041cbd6b634c7083d035ea8d8da73e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1730983327295 2024-11-07T12:42:09,439 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 254b0d38fd9a4fcf80b0a0e3fcc72b82, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1730983327295 2024-11-07T12:42:09,439 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 240af32ce2724ad996f2a04c71cb3710, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1730983328516 2024-11-07T12:42:09,439 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 68e78311dd5540afb6c30605edae96be, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1730983328516 2024-11-07T12:42:09,464 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8dc13886c19c8110f075e43282960321#A#compaction#204 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:42:09,465 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/3b3c7ae72d1f4c90939fd94c1a32d8cc is 50, key is test_row_0/A:col10/1730983328521/Put/seqid=0 2024-11-07T12:42:09,472 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8dc13886c19c8110f075e43282960321#B#compaction#205 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:42:09,473 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/e8e2bcccd7a14b779db96f8bc532ef96 is 50, key is test_row_0/B:col10/1730983328521/Put/seqid=0 2024-11-07T12:42:09,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742078_1254 (size=12104) 2024-11-07T12:42:09,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742079_1255 (size=12104) 2024-11-07T12:42:09,516 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/e8e2bcccd7a14b779db96f8bc532ef96 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/e8e2bcccd7a14b779db96f8bc532ef96 2024-11-07T12:42:09,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 8dc13886c19c8110f075e43282960321 2024-11-07T12:42:09,525 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8dc13886c19c8110f075e43282960321 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-07T12:42:09,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=A 2024-11-07T12:42:09,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:09,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=B 2024-11-07T12:42:09,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:09,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=C 2024-11-07T12:42:09,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:09,528 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8dc13886c19c8110f075e43282960321/B of 8dc13886c19c8110f075e43282960321 into e8e2bcccd7a14b779db96f8bc532ef96(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:42:09,528 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8dc13886c19c8110f075e43282960321: 2024-11-07T12:42:09,528 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., storeName=8dc13886c19c8110f075e43282960321/B, priority=13, startTime=1730983329435; duration=0sec 2024-11-07T12:42:09,528 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:42:09,528 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8dc13886c19c8110f075e43282960321:B 2024-11-07T12:42:09,528 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:42:09,529 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:42:09,529 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 8dc13886c19c8110f075e43282960321/C is initiating minor compaction (all files) 2024-11-07T12:42:09,529 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8dc13886c19c8110f075e43282960321/C in TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:09,530 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/07f8545df3d647dfb97011bf2da37603, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/883d4efb87a34329bc3324daa3403b1a, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/06c86adf7f6342a2b61c285712d237d0] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp, totalSize=35.2 K 2024-11-07T12:42:09,530 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 07f8545df3d647dfb97011bf2da37603, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1730983327262 2024-11-07T12:42:09,530 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 883d4efb87a34329bc3324daa3403b1a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1730983327295 2024-11-07T12:42:09,531 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 06c86adf7f6342a2b61c285712d237d0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1730983328516 2024-11-07T12:42:09,531 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/fd2e5fefe2a646e7a29a6913a01a422a is 50, key is test_row_0/A:col10/1730983328879/Put/seqid=0 2024-11-07T12:42:09,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-07T12:42:09,543 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8dc13886c19c8110f075e43282960321#C#compaction#207 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:42:09,544 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/9faf3b34597640e38adc0ec9855c08a0 is 50, key is test_row_0/C:col10/1730983328521/Put/seqid=0 2024-11-07T12:42:09,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:09,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42026 deadline: 1730983389543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:09,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:09,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42054 deadline: 1730983389544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:09,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:09,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:09,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:09,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:09,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:09,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:09,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:09,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41976 deadline: 1730983389546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:09,552 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:09,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:09,552 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:09,552 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:09,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42034 deadline: 1730983389546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:09,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:09,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:09,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:09,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:09,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42070 deadline: 1730983389547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:09,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:09,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:09,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:09,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742080_1256 (size=12001) 2024-11-07T12:42:09,563 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=67 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/fd2e5fefe2a646e7a29a6913a01a422a 2024-11-07T12:42:09,575 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-07T12:42:09,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:09,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. as already flushing 2024-11-07T12:42:09,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:09,576 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:09,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:09,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:09,577 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/945a9355e07a4ac8ab5c9126bfd06075 is 50, key is test_row_0/B:col10/1730983328879/Put/seqid=0 2024-11-07T12:42:09,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742081_1257 (size=12104) 2024-11-07T12:42:09,612 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/9faf3b34597640e38adc0ec9855c08a0 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/9faf3b34597640e38adc0ec9855c08a0 2024-11-07T12:42:09,619 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8dc13886c19c8110f075e43282960321/C of 8dc13886c19c8110f075e43282960321 into 9faf3b34597640e38adc0ec9855c08a0(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:42:09,619 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8dc13886c19c8110f075e43282960321: 2024-11-07T12:42:09,619 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., storeName=8dc13886c19c8110f075e43282960321/C, priority=13, startTime=1730983329435; duration=0sec 2024-11-07T12:42:09,619 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:42:09,619 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8dc13886c19c8110f075e43282960321:C 2024-11-07T12:42:09,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742082_1258 (size=12001) 2024-11-07T12:42:09,632 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=67 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/945a9355e07a4ac8ab5c9126bfd06075 2024-11-07T12:42:09,643 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/eb67a0705e3048beb64e6920ab33f7aa is 50, key is test_row_0/C:col10/1730983328879/Put/seqid=0 2024-11-07T12:42:09,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742083_1259 (size=12001) 2024-11-07T12:42:09,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:09,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41976 deadline: 1730983389662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:09,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:09,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42034 deadline: 1730983389662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:09,667 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:09,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:09,667 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:09,667 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:09,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42026 deadline: 1730983389662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:09,668 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:09,668 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:09,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:09,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:09,668 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:09,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:09,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42070 deadline: 1730983389662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:09,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:09,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:09,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:09,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:09,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:09,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42054 deadline: 1730983389662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:09,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:09,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:09,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:09,729 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-07T12:42:09,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:09,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. as already flushing 2024-11-07T12:42:09,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:09,730 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:09,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:09,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:09,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-07T12:42:09,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:09,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41976 deadline: 1730983389872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:09,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:09,875 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:09,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42034 deadline: 1730983389872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:09,875 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:09,875 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:09,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:09,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42026 deadline: 1730983389872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:09,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:09,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:09,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:09,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:09,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:09,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:09,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42070 deadline: 1730983389872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:09,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:09,876 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:09,876 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:09,876 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:09,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:09,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42054 deadline: 1730983389881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:09,885 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:09,885 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:09,885 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:09,888 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-07T12:42:09,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:09,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. as already flushing 2024-11-07T12:42:09,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:09,888 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:09,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:09,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:09,911 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/3b3c7ae72d1f4c90939fd94c1a32d8cc as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/3b3c7ae72d1f4c90939fd94c1a32d8cc 2024-11-07T12:42:09,924 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8dc13886c19c8110f075e43282960321/A of 8dc13886c19c8110f075e43282960321 into 3b3c7ae72d1f4c90939fd94c1a32d8cc(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:42:09,924 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8dc13886c19c8110f075e43282960321: 2024-11-07T12:42:09,924 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., storeName=8dc13886c19c8110f075e43282960321/A, priority=13, startTime=1730983329435; duration=0sec 2024-11-07T12:42:09,924 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:42:09,924 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8dc13886c19c8110f075e43282960321:A 2024-11-07T12:42:10,044 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-07T12:42:10,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:10,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. as already flushing 2024-11-07T12:42:10,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:10,044 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:10,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:10,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:10,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-07T12:42:10,059 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=67 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/eb67a0705e3048beb64e6920ab33f7aa 2024-11-07T12:42:10,064 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/fd2e5fefe2a646e7a29a6913a01a422a as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/fd2e5fefe2a646e7a29a6913a01a422a 2024-11-07T12:42:10,070 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/fd2e5fefe2a646e7a29a6913a01a422a, entries=150, sequenceid=67, filesize=11.7 K 2024-11-07T12:42:10,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/945a9355e07a4ac8ab5c9126bfd06075 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/945a9355e07a4ac8ab5c9126bfd06075 2024-11-07T12:42:10,075 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/945a9355e07a4ac8ab5c9126bfd06075, entries=150, sequenceid=67, filesize=11.7 K 2024-11-07T12:42:10,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/eb67a0705e3048beb64e6920ab33f7aa as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/eb67a0705e3048beb64e6920ab33f7aa 2024-11-07T12:42:10,082 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/eb67a0705e3048beb64e6920ab33f7aa, entries=150, sequenceid=67, filesize=11.7 K 2024-11-07T12:42:10,082 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 8dc13886c19c8110f075e43282960321 in 558ms, sequenceid=67, compaction requested=false 2024-11-07T12:42:10,085 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8dc13886c19c8110f075e43282960321: 2024-11-07T12:42:10,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 8dc13886c19c8110f075e43282960321 2024-11-07T12:42:10,186 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8dc13886c19c8110f075e43282960321 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-07T12:42:10,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=A 2024-11-07T12:42:10,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:10,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=B 2024-11-07T12:42:10,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:10,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=C 2024-11-07T12:42:10,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:10,198 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-07T12:42:10,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:10,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. as already flushing 2024-11-07T12:42:10,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:10,198 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:10,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:10,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:10,211 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/551c2dbdd1ee47e9850fe0ad64a35d55 is 50, key is test_row_0/A:col10/1730983330184/Put/seqid=0 2024-11-07T12:42:10,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:10,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41976 deadline: 1730983390212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:10,222 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:10,222 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:10,222 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:10,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:10,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42034 deadline: 1730983390214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:10,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:10,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42054 deadline: 1730983390215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:10,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:10,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:10,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:10,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:10,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:10,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:10,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:10,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42026 deadline: 1730983390222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:10,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:10,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:10,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:10,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:10,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42070 deadline: 1730983390235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:10,241 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:10,241 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:10,241 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:10,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742084_1260 (size=12001) 2024-11-07T12:42:10,267 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/551c2dbdd1ee47e9850fe0ad64a35d55 2024-11-07T12:42:10,279 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/de80696a81c943188a3a32024a92b8b8 is 50, key is test_row_0/B:col10/1730983330184/Put/seqid=0 2024-11-07T12:42:10,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742085_1261 (size=12001) 2024-11-07T12:42:10,305 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/de80696a81c943188a3a32024a92b8b8 2024-11-07T12:42:10,324 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/6d94293b6e0c4b9a8d688d746be6a135 is 50, key is test_row_0/C:col10/1730983330184/Put/seqid=0 2024-11-07T12:42:10,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:10,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41976 deadline: 1730983390331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:10,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:10,337 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:10,337 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:10,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42034 deadline: 1730983390331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:10,337 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:10,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:10,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42054 deadline: 1730983390331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:10,337 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:10,337 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:10,337 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:10,338 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:10,338 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:10,338 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:10,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:10,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42026 deadline: 1730983390342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:10,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:10,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:10,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:10,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742086_1262 (size=12001) 2024-11-07T12:42:10,352 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-07T12:42:10,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:10,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. as already flushing 2024-11-07T12:42:10,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:10,352 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:10,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:10,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:10,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:10,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42070 deadline: 1730983390352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:10,356 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:10,356 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:10,356 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:10,505 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-07T12:42:10,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:10,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. as already flushing 2024-11-07T12:42:10,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:10,506 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:10,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:10,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:10,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:10,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41976 deadline: 1730983390542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:10,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:10,548 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:10,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42054 deadline: 1730983390542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:10,548 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:10,548 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:10,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:10,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:10,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42034 deadline: 1730983390543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:10,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:10,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:10,549 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:10,549 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:10,549 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:10,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:10,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42026 deadline: 1730983390552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:10,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:10,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:10,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:10,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-07T12:42:10,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:10,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42070 deadline: 1730983390562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:10,566 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:10,566 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:10,566 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:10,665 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-07T12:42:10,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:10,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. as already flushing 2024-11-07T12:42:10,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:10,666 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:10,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:10,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:10,750 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/6d94293b6e0c4b9a8d688d746be6a135 2024-11-07T12:42:10,757 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/551c2dbdd1ee47e9850fe0ad64a35d55 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/551c2dbdd1ee47e9850fe0ad64a35d55 2024-11-07T12:42:10,763 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/551c2dbdd1ee47e9850fe0ad64a35d55, entries=150, sequenceid=86, filesize=11.7 K 2024-11-07T12:42:10,763 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/de80696a81c943188a3a32024a92b8b8 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/de80696a81c943188a3a32024a92b8b8 2024-11-07T12:42:10,771 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/de80696a81c943188a3a32024a92b8b8, entries=150, sequenceid=86, filesize=11.7 K 2024-11-07T12:42:10,773 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/6d94293b6e0c4b9a8d688d746be6a135 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/6d94293b6e0c4b9a8d688d746be6a135 2024-11-07T12:42:10,783 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/6d94293b6e0c4b9a8d688d746be6a135, entries=150, sequenceid=86, filesize=11.7 K 2024-11-07T12:42:10,784 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 8dc13886c19c8110f075e43282960321 in 598ms, sequenceid=86, compaction requested=true 2024-11-07T12:42:10,784 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8dc13886c19c8110f075e43282960321: 2024-11-07T12:42:10,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8dc13886c19c8110f075e43282960321:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:42:10,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:42:10,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8dc13886c19c8110f075e43282960321:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:42:10,785 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:42:10,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:42:10,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8dc13886c19c8110f075e43282960321:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:42:10,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:42:10,785 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:42:10,788 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:42:10,788 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): 8dc13886c19c8110f075e43282960321/A is initiating minor compaction (all files) 2024-11-07T12:42:10,788 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8dc13886c19c8110f075e43282960321/A in TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:10,788 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/3b3c7ae72d1f4c90939fd94c1a32d8cc, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/fd2e5fefe2a646e7a29a6913a01a422a, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/551c2dbdd1ee47e9850fe0ad64a35d55] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp, totalSize=35.3 K 2024-11-07T12:42:10,789 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:42:10,789 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 8dc13886c19c8110f075e43282960321/B is initiating minor compaction (all files) 2024-11-07T12:42:10,789 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8dc13886c19c8110f075e43282960321/B in TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:10,789 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/e8e2bcccd7a14b779db96f8bc532ef96, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/945a9355e07a4ac8ab5c9126bfd06075, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/de80696a81c943188a3a32024a92b8b8] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp, totalSize=35.3 K 2024-11-07T12:42:10,789 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b3c7ae72d1f4c90939fd94c1a32d8cc, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1730983328516 2024-11-07T12:42:10,790 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting e8e2bcccd7a14b779db96f8bc532ef96, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1730983328516 2024-11-07T12:42:10,790 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd2e5fefe2a646e7a29a6913a01a422a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1730983328879 2024-11-07T12:42:10,790 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 945a9355e07a4ac8ab5c9126bfd06075, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1730983328879 2024-11-07T12:42:10,791 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 551c2dbdd1ee47e9850fe0ad64a35d55, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1730983330184 2024-11-07T12:42:10,791 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting de80696a81c943188a3a32024a92b8b8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1730983330184 2024-11-07T12:42:10,802 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8dc13886c19c8110f075e43282960321#A#compaction#213 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:42:10,802 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/3c2bdc66274549b1b9220c4982e484ca is 50, key is test_row_0/A:col10/1730983330184/Put/seqid=0 2024-11-07T12:42:10,815 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8dc13886c19c8110f075e43282960321#B#compaction#214 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:42:10,816 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/65ccef9dff8d482f973b665056dac44f is 50, key is test_row_0/B:col10/1730983330184/Put/seqid=0 2024-11-07T12:42:10,844 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-07T12:42:10,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:10,844 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 8dc13886c19c8110f075e43282960321 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-07T12:42:10,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=A 2024-11-07T12:42:10,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:10,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=B 2024-11-07T12:42:10,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:10,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=C 2024-11-07T12:42:10,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:10,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/0e12360be28f435da4ddefb2996a9077 is 50, key is test_row_0/A:col10/1730983330197/Put/seqid=0 2024-11-07T12:42:10,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742087_1263 (size=12207) 2024-11-07T12:42:10,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 8dc13886c19c8110f075e43282960321 2024-11-07T12:42:10,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. as already flushing 2024-11-07T12:42:10,871 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/3c2bdc66274549b1b9220c4982e484ca as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/3c2bdc66274549b1b9220c4982e484ca 2024-11-07T12:42:10,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742088_1264 (size=12207) 2024-11-07T12:42:10,881 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8dc13886c19c8110f075e43282960321/A of 8dc13886c19c8110f075e43282960321 into 3c2bdc66274549b1b9220c4982e484ca(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:42:10,881 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8dc13886c19c8110f075e43282960321: 2024-11-07T12:42:10,881 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., storeName=8dc13886c19c8110f075e43282960321/A, priority=13, startTime=1730983330784; duration=0sec 2024-11-07T12:42:10,881 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:42:10,881 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8dc13886c19c8110f075e43282960321:A 2024-11-07T12:42:10,881 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:42:10,883 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/65ccef9dff8d482f973b665056dac44f as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/65ccef9dff8d482f973b665056dac44f 2024-11-07T12:42:10,884 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:42:10,884 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): 8dc13886c19c8110f075e43282960321/C is initiating minor compaction (all files) 2024-11-07T12:42:10,884 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8dc13886c19c8110f075e43282960321/C in TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:10,884 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/9faf3b34597640e38adc0ec9855c08a0, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/eb67a0705e3048beb64e6920ab33f7aa, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/6d94293b6e0c4b9a8d688d746be6a135] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp, totalSize=35.3 K 2024-11-07T12:42:10,885 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9faf3b34597640e38adc0ec9855c08a0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1730983328516 2024-11-07T12:42:10,887 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb67a0705e3048beb64e6920ab33f7aa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1730983328879 2024-11-07T12:42:10,889 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d94293b6e0c4b9a8d688d746be6a135, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1730983330184 2024-11-07T12:42:10,890 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8dc13886c19c8110f075e43282960321/B of 8dc13886c19c8110f075e43282960321 into 65ccef9dff8d482f973b665056dac44f(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:42:10,890 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8dc13886c19c8110f075e43282960321: 2024-11-07T12:42:10,890 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., storeName=8dc13886c19c8110f075e43282960321/B, priority=13, startTime=1730983330784; duration=0sec 2024-11-07T12:42:10,891 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:42:10,891 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8dc13886c19c8110f075e43282960321:B 2024-11-07T12:42:10,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:10,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42054 deadline: 1730983390887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:10,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:10,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42070 deadline: 1730983390888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:10,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:10,900 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:10,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:10,900 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:10,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:10,900 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:10,901 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:10,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42034 deadline: 1730983390890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:10,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:10,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42026 deadline: 1730983390891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:10,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:10,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41976 deadline: 1730983390901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:10,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:10,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:10,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:10,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:10,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:10,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:10,904 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8dc13886c19c8110f075e43282960321#C#compaction#216 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:42:10,904 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:10,904 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:10,904 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:10,905 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/7ea3b631081a49a28ef38de44c3aac48 is 50, key is test_row_0/C:col10/1730983330184/Put/seqid=0 2024-11-07T12:42:10,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742089_1265 (size=12001) 2024-11-07T12:42:10,907 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/0e12360be28f435da4ddefb2996a9077 2024-11-07T12:42:10,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/a0f88cc157694d179447a4d2aba24fc1 is 50, key is test_row_0/B:col10/1730983330197/Put/seqid=0 2024-11-07T12:42:10,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742090_1266 (size=12207) 2024-11-07T12:42:10,965 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/7ea3b631081a49a28ef38de44c3aac48 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/7ea3b631081a49a28ef38de44c3aac48 2024-11-07T12:42:10,975 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8dc13886c19c8110f075e43282960321/C of 8dc13886c19c8110f075e43282960321 into 7ea3b631081a49a28ef38de44c3aac48(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:42:10,975 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8dc13886c19c8110f075e43282960321: 2024-11-07T12:42:10,975 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., storeName=8dc13886c19c8110f075e43282960321/C, priority=13, startTime=1730983330785; duration=0sec 2024-11-07T12:42:10,975 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:42:10,975 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8dc13886c19c8110f075e43282960321:C 2024-11-07T12:42:10,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742091_1267 (size=12001) 2024-11-07T12:42:11,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:11,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42054 deadline: 1730983391002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:11,004 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:11,004 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:11,004 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:11,004 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:11,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42070 deadline: 1730983391002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:11,004 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:11,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:11,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:11,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:11,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42026 deadline: 1730983391011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:11,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:11,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:11,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:11,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:11,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41976 deadline: 1730983391012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:11,015 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:11,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:11,015 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:11,015 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:11,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42034 deadline: 1730983391013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:11,016 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:11,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:11,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:11,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:11,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42054 deadline: 1730983391211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:11,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:11,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:11,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42070 deadline: 1730983391211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:11,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:11,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:11,214 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:11,214 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:11,214 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:11,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:11,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42034 deadline: 1730983391222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:11,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:11,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42026 deadline: 1730983391222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:11,225 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:11,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41976 deadline: 1730983391224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:11,226 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:11,226 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:11,226 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:11,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:11,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:11,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:11,231 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:11,231 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:11,231 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:11,379 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/a0f88cc157694d179447a4d2aba24fc1 2024-11-07T12:42:11,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/2da541cdae284a21bf92afb554eab79b is 50, key is test_row_0/C:col10/1730983330197/Put/seqid=0 2024-11-07T12:42:11,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742092_1268 (size=12001) 2024-11-07T12:42:11,399 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/2da541cdae284a21bf92afb554eab79b 2024-11-07T12:42:11,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/0e12360be28f435da4ddefb2996a9077 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/0e12360be28f435da4ddefb2996a9077 2024-11-07T12:42:11,410 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/0e12360be28f435da4ddefb2996a9077, entries=150, sequenceid=103, filesize=11.7 K 2024-11-07T12:42:11,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/a0f88cc157694d179447a4d2aba24fc1 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/a0f88cc157694d179447a4d2aba24fc1 2024-11-07T12:42:11,417 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/a0f88cc157694d179447a4d2aba24fc1, entries=150, sequenceid=103, filesize=11.7 K 2024-11-07T12:42:11,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/2da541cdae284a21bf92afb554eab79b as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/2da541cdae284a21bf92afb554eab79b 2024-11-07T12:42:11,429 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/2da541cdae284a21bf92afb554eab79b, entries=150, sequenceid=103, filesize=11.7 K 2024-11-07T12:42:11,431 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 8dc13886c19c8110f075e43282960321 in 587ms, sequenceid=103, compaction requested=false 2024-11-07T12:42:11,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 8dc13886c19c8110f075e43282960321: 2024-11-07T12:42:11,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:11,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-07T12:42:11,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-07T12:42:11,436 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-07T12:42:11,436 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0110 sec 2024-11-07T12:42:11,440 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 2.0180 sec 2024-11-07T12:42:11,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 8dc13886c19c8110f075e43282960321 2024-11-07T12:42:11,534 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8dc13886c19c8110f075e43282960321 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-07T12:42:11,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=A 2024-11-07T12:42:11,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:11,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=B 2024-11-07T12:42:11,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:11,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=C 2024-11-07T12:42:11,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:11,559 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/79dd3052429841f1b5380ea68d0a7be1 is 50, key is test_row_0/A:col10/1730983331526/Put/seqid=0 2024-11-07T12:42:11,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:11,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42034 deadline: 1730983391560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:11,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:11,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:11,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:11,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:11,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42054 deadline: 1730983391562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:11,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:11,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:11,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:11,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-07T12:42:11,574 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:42:11,575 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:42:11,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:11,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42070 deadline: 1730983391569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:11,579 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:11,579 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:11,579 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:11,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:11,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41976 deadline: 1730983391573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:11,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:11,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42026 deadline: 1730983391573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:11,582 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:11,582 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:11,582 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:11,582 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:11,582 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:11,582 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:11,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-07T12:42:11,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-07T12:42:11,600 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:42:11,601 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:42:11,601 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:42:11,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742093_1269 (size=12001) 2024-11-07T12:42:11,623 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=123 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/79dd3052429841f1b5380ea68d0a7be1 2024-11-07T12:42:11,669 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/097d432acecf42118cfeee6ae22347f1 is 50, key is test_row_0/B:col10/1730983331526/Put/seqid=0 2024-11-07T12:42:11,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:11,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42034 deadline: 1730983391672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:11,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:11,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:11,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:11,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:11,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42054 deadline: 1730983391684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:11,690 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:11,690 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:11,690 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:11,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:11,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42070 deadline: 1730983391684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:11,690 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:11,691 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:11,691 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:11,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-07T12:42:11,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:11,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42026 deadline: 1730983391693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:11,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:11,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41976 deadline: 1730983391693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:11,700 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:11,701 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:11,701 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:11,701 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:11,701 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:11,701 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:11,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742094_1270 (size=12001) 2024-11-07T12:42:11,757 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-07T12:42:11,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:11,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. as already flushing 2024-11-07T12:42:11,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:11,757 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:11,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:11,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:11,765 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T12:42:11,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:11,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42034 deadline: 1730983391882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:11,895 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:11,895 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:11,895 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:11,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-07T12:42:11,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:11,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42054 deadline: 1730983391903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:11,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:11,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:11,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:11,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:11,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42070 deadline: 1730983391903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:11,911 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:11,912 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:11,912 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:11,913 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-07T12:42:11,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:11,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. as already flushing 2024-11-07T12:42:11,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:11,914 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:11,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:11,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:11,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:11,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41976 deadline: 1730983391912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:11,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:11,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42026 deadline: 1730983391912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:11,917 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:11,917 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:11,917 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:11,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:11,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:11,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:12,072 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-07T12:42:12,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:12,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. as already flushing 2024-11-07T12:42:12,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:12,073 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:12,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:12,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:12,105 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=123 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/097d432acecf42118cfeee6ae22347f1 2024-11-07T12:42:12,122 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/4d4c03d2659d46d4b0c5e71684b866b0 is 50, key is test_row_0/C:col10/1730983331526/Put/seqid=0 2024-11-07T12:42:12,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742095_1271 (size=12001) 2024-11-07T12:42:12,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:12,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42034 deadline: 1730983392202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:12,204 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:12,205 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:12,205 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:12,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-07T12:42:12,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:12,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42054 deadline: 1730983392212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:12,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:12,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:12,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:12,225 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:12,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42026 deadline: 1730983392222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:12,225 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:12,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42070 deadline: 1730983392222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:12,225 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:12,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41976 deadline: 1730983392222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:12,229 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-07T12:42:12,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:12,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. as already flushing 2024-11-07T12:42:12,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:12,229 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:12,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:12,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:12,231 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:12,231 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:12,231 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:12,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:12,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:12,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:12,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:12,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:12,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:12,383 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-07T12:42:12,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:12,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. as already flushing 2024-11-07T12:42:12,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:12,384 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:12,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:12,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:12,537 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=123 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/4d4c03d2659d46d4b0c5e71684b866b0 2024-11-07T12:42:12,541 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-07T12:42:12,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:12,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. as already flushing 2024-11-07T12:42:12,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:12,541 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:12,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:12,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:12,546 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/79dd3052429841f1b5380ea68d0a7be1 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/79dd3052429841f1b5380ea68d0a7be1 2024-11-07T12:42:12,553 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/79dd3052429841f1b5380ea68d0a7be1, entries=150, sequenceid=123, filesize=11.7 K 2024-11-07T12:42:12,554 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/097d432acecf42118cfeee6ae22347f1 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/097d432acecf42118cfeee6ae22347f1 2024-11-07T12:42:12,558 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/097d432acecf42118cfeee6ae22347f1, entries=150, sequenceid=123, filesize=11.7 K 2024-11-07T12:42:12,559 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/4d4c03d2659d46d4b0c5e71684b866b0 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/4d4c03d2659d46d4b0c5e71684b866b0 2024-11-07T12:42:12,563 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/4d4c03d2659d46d4b0c5e71684b866b0, entries=150, sequenceid=123, filesize=11.7 K 2024-11-07T12:42:12,564 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 8dc13886c19c8110f075e43282960321 in 1030ms, sequenceid=123, compaction requested=true 2024-11-07T12:42:12,564 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8dc13886c19c8110f075e43282960321: 2024-11-07T12:42:12,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8dc13886c19c8110f075e43282960321:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:42:12,564 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:42:12,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:42:12,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8dc13886c19c8110f075e43282960321:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:42:12,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:42:12,564 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:42:12,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8dc13886c19c8110f075e43282960321:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:42:12,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:42:12,565 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:42:12,565 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): 8dc13886c19c8110f075e43282960321/A is initiating minor compaction (all files) 2024-11-07T12:42:12,565 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8dc13886c19c8110f075e43282960321/A in TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:12,566 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/3c2bdc66274549b1b9220c4982e484ca, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/0e12360be28f435da4ddefb2996a9077, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/79dd3052429841f1b5380ea68d0a7be1] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp, totalSize=35.4 K 2024-11-07T12:42:12,566 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:42:12,566 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 8dc13886c19c8110f075e43282960321/B is initiating minor compaction (all files) 2024-11-07T12:42:12,566 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8dc13886c19c8110f075e43282960321/B in TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:12,566 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/65ccef9dff8d482f973b665056dac44f, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/a0f88cc157694d179447a4d2aba24fc1, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/097d432acecf42118cfeee6ae22347f1] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp, totalSize=35.4 K 2024-11-07T12:42:12,566 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c2bdc66274549b1b9220c4982e484ca, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1730983330184 2024-11-07T12:42:12,567 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 65ccef9dff8d482f973b665056dac44f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1730983330184 2024-11-07T12:42:12,567 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e12360be28f435da4ddefb2996a9077, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1730983330197 2024-11-07T12:42:12,567 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting a0f88cc157694d179447a4d2aba24fc1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1730983330197 2024-11-07T12:42:12,568 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79dd3052429841f1b5380ea68d0a7be1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1730983330895 2024-11-07T12:42:12,568 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 097d432acecf42118cfeee6ae22347f1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1730983330895 2024-11-07T12:42:12,583 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8dc13886c19c8110f075e43282960321#B#compaction#222 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:42:12,583 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/f1f9e3ca46574b6fba6bfd1e572040ea is 50, key is test_row_0/B:col10/1730983331526/Put/seqid=0 2024-11-07T12:42:12,603 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8dc13886c19c8110f075e43282960321#A#compaction#223 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:42:12,604 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/f4c22dffcd63417cba781ba4a7661ef6 is 50, key is test_row_0/A:col10/1730983331526/Put/seqid=0 2024-11-07T12:42:12,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742096_1272 (size=12309) 2024-11-07T12:42:12,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742097_1273 (size=12309) 2024-11-07T12:42:12,680 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/f4c22dffcd63417cba781ba4a7661ef6 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/f4c22dffcd63417cba781ba4a7661ef6 2024-11-07T12:42:12,689 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8dc13886c19c8110f075e43282960321/A of 8dc13886c19c8110f075e43282960321 into f4c22dffcd63417cba781ba4a7661ef6(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:42:12,689 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8dc13886c19c8110f075e43282960321: 2024-11-07T12:42:12,689 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., storeName=8dc13886c19c8110f075e43282960321/A, priority=13, startTime=1730983332564; duration=0sec 2024-11-07T12:42:12,690 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:42:12,690 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8dc13886c19c8110f075e43282960321:A 2024-11-07T12:42:12,690 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:42:12,692 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:42:12,692 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): 8dc13886c19c8110f075e43282960321/C is initiating minor compaction (all files) 2024-11-07T12:42:12,692 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8dc13886c19c8110f075e43282960321/C in TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:12,692 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/7ea3b631081a49a28ef38de44c3aac48, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/2da541cdae284a21bf92afb554eab79b, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/4d4c03d2659d46d4b0c5e71684b866b0] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp, totalSize=35.4 K 2024-11-07T12:42:12,693 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ea3b631081a49a28ef38de44c3aac48, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1730983330184 2024-11-07T12:42:12,693 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2da541cdae284a21bf92afb554eab79b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1730983330197 2024-11-07T12:42:12,693 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d4c03d2659d46d4b0c5e71684b866b0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1730983330895 2024-11-07T12:42:12,695 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-07T12:42:12,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:12,696 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 8dc13886c19c8110f075e43282960321 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-07T12:42:12,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=A 2024-11-07T12:42:12,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:12,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=B 2024-11-07T12:42:12,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:12,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=C 2024-11-07T12:42:12,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:12,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 8dc13886c19c8110f075e43282960321 2024-11-07T12:42:12,715 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. as already flushing 2024-11-07T12:42:12,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/028e09b4fe5346239a93a80b44fb8fea is 50, key is test_row_0/A:col10/1730983331567/Put/seqid=0 2024-11-07T12:42:12,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-07T12:42:12,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:12,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41976 deadline: 1730983392752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:12,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:12,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742098_1274 (size=12151) 2024-11-07T12:42:12,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42070 deadline: 1730983392752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:12,762 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:12,762 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:12,762 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:12,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:12,762 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:12,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42034 deadline: 1730983392752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:12,762 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:12,762 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:12,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:12,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:12,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:12,764 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/028e09b4fe5346239a93a80b44fb8fea 2024-11-07T12:42:12,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:12,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42054 deadline: 1730983392760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:12,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:12,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:12,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:12,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:12,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42026 deadline: 1730983392762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:12,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:12,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:12,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:12,771 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8dc13886c19c8110f075e43282960321#C#compaction#225 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:42:12,771 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/0d2ec7477807410385f2f33995e21377 is 50, key is test_row_0/C:col10/1730983331526/Put/seqid=0 2024-11-07T12:42:12,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/b4ed6e537731444b8c90a0bf75f8773c is 50, key is test_row_0/B:col10/1730983331567/Put/seqid=0 2024-11-07T12:42:12,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742099_1275 (size=12309) 2024-11-07T12:42:12,797 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/0d2ec7477807410385f2f33995e21377 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/0d2ec7477807410385f2f33995e21377 2024-11-07T12:42:12,804 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8dc13886c19c8110f075e43282960321/C of 8dc13886c19c8110f075e43282960321 into 0d2ec7477807410385f2f33995e21377(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:42:12,804 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8dc13886c19c8110f075e43282960321: 2024-11-07T12:42:12,804 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., storeName=8dc13886c19c8110f075e43282960321/C, priority=13, startTime=1730983332564; duration=0sec 2024-11-07T12:42:12,805 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:42:12,805 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8dc13886c19c8110f075e43282960321:C 2024-11-07T12:42:12,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742100_1276 (size=12151) 2024-11-07T12:42:12,835 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/b4ed6e537731444b8c90a0bf75f8773c 2024-11-07T12:42:12,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/d2b18571482a46be9d0cb6c95864a1ec is 50, key is test_row_0/C:col10/1730983331567/Put/seqid=0 2024-11-07T12:42:12,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742101_1277 (size=12151) 2024-11-07T12:42:12,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:12,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41976 deadline: 1730983392871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:12,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:12,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42070 deadline: 1730983392872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:12,875 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:12,875 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:12,875 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:12,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:12,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42034 deadline: 1730983392872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:12,876 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:12,876 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:12,876 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:12,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:12,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:12,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:12,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42054 deadline: 1730983392872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:12,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:12,876 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:12,876 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:12,876 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:12,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:12,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42026 deadline: 1730983392882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:12,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:12,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:12,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:13,050 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/f1f9e3ca46574b6fba6bfd1e572040ea as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/f1f9e3ca46574b6fba6bfd1e572040ea 2024-11-07T12:42:13,058 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8dc13886c19c8110f075e43282960321/B of 8dc13886c19c8110f075e43282960321 into f1f9e3ca46574b6fba6bfd1e572040ea(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:42:13,058 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8dc13886c19c8110f075e43282960321: 2024-11-07T12:42:13,059 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., storeName=8dc13886c19c8110f075e43282960321/B, priority=13, startTime=1730983332564; duration=0sec 2024-11-07T12:42:13,060 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:42:13,060 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8dc13886c19c8110f075e43282960321:B 2024-11-07T12:42:13,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:13,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41976 deadline: 1730983393083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:13,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:13,087 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:13,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:13,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42054 deadline: 1730983393083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:13,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:13,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:13,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42034 deadline: 1730983393083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:13,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:13,089 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:13,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42070 deadline: 1730983393084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:13,089 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:13,089 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:13,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:13,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:13,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:13,089 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:13,089 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:13,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:13,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:13,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42026 deadline: 1730983393092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:13,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:13,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:13,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:13,252 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/d2b18571482a46be9d0cb6c95864a1ec 2024-11-07T12:42:13,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/028e09b4fe5346239a93a80b44fb8fea as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/028e09b4fe5346239a93a80b44fb8fea 2024-11-07T12:42:13,264 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/028e09b4fe5346239a93a80b44fb8fea, entries=150, sequenceid=139, filesize=11.9 K 2024-11-07T12:42:13,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/b4ed6e537731444b8c90a0bf75f8773c as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/b4ed6e537731444b8c90a0bf75f8773c 2024-11-07T12:42:13,275 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/b4ed6e537731444b8c90a0bf75f8773c, entries=150, sequenceid=139, filesize=11.9 K 2024-11-07T12:42:13,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/d2b18571482a46be9d0cb6c95864a1ec as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/d2b18571482a46be9d0cb6c95864a1ec 2024-11-07T12:42:13,294 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/d2b18571482a46be9d0cb6c95864a1ec, entries=150, sequenceid=139, filesize=11.9 K 2024-11-07T12:42:13,295 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 8dc13886c19c8110f075e43282960321 in 599ms, sequenceid=139, compaction requested=false 2024-11-07T12:42:13,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 8dc13886c19c8110f075e43282960321: 2024-11-07T12:42:13,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:13,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-07T12:42:13,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-07T12:42:13,300 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-07T12:42:13,300 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6970 sec 2024-11-07T12:42:13,315 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.7290 sec 2024-11-07T12:42:13,406 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8dc13886c19c8110f075e43282960321 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-07T12:42:13,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 8dc13886c19c8110f075e43282960321 2024-11-07T12:42:13,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=A 2024-11-07T12:42:13,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:13,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=B 2024-11-07T12:42:13,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:13,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=C 2024-11-07T12:42:13,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:13,425 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/3a9f823b96a044fe8612cb11c76313b2 is 50, key is test_row_0/A:col10/1730983332750/Put/seqid=0 2024-11-07T12:42:13,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:13,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42054 deadline: 1730983393428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:13,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:13,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41976 deadline: 1730983393428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:13,436 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:13,436 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:13,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:13,436 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:13,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42026 deadline: 1730983393429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:13,436 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:13,436 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:13,437 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:13,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:13,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:13,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42070 deadline: 1730983393430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:13,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:13,438 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:13,438 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:13,438 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:13,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:13,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:13,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42034 deadline: 1730983393435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:13,441 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:13,441 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:13,441 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:13,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742102_1278 (size=12151) 2024-11-07T12:42:13,477 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/3a9f823b96a044fe8612cb11c76313b2 2024-11-07T12:42:13,495 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/33948893b9364b75a3a7b522f3e2a5d6 is 50, key is test_row_0/B:col10/1730983332750/Put/seqid=0 2024-11-07T12:42:13,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:13,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42054 deadline: 1730983393542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:13,544 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:13,544 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:13,544 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:13,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742103_1279 (size=12151) 2024-11-07T12:42:13,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:13,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42070 deadline: 1730983393543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:13,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:13,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:13,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41976 deadline: 1730983393543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:13,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42026 deadline: 1730983393543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:13,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:13,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:13,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:13,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:13,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:13,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:13,551 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:13,551 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:13,551 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:13,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:13,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42034 deadline: 1730983393552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:13,556 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:13,556 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:13,556 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:13,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-07T12:42:13,732 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:42:13,733 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:42:13,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-07T12:42:13,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-07T12:42:13,748 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:42:13,753 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:42:13,753 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:42:13,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:13,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42054 deadline: 1730983393752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:13,758 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:13,758 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:13,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:13,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:13,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41976 deadline: 1730983393762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:13,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:13,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:13,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42070 deadline: 1730983393762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:13,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42034 deadline: 1730983393762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:13,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:13,766 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:13,766 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:13,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42026 deadline: 1730983393762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:13,766 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:13,766 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:13,766 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:13,766 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:13,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:13,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:13,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:13,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:13,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:13,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:13,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-07T12:42:13,907 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-07T12:42:13,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:13,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. as already flushing 2024-11-07T12:42:13,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:13,908 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:13,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:13,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:13,948 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/33948893b9364b75a3a7b522f3e2a5d6 2024-11-07T12:42:13,963 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/7ca0ade2a2f4482682d7474b13fb86a2 is 50, key is test_row_0/C:col10/1730983332750/Put/seqid=0 2024-11-07T12:42:13,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742104_1280 (size=12151) 2024-11-07T12:42:13,996 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/7ca0ade2a2f4482682d7474b13fb86a2 2024-11-07T12:42:14,001 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/3a9f823b96a044fe8612cb11c76313b2 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/3a9f823b96a044fe8612cb11c76313b2 2024-11-07T12:42:14,006 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/3a9f823b96a044fe8612cb11c76313b2, entries=150, sequenceid=161, filesize=11.9 K 2024-11-07T12:42:14,009 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/33948893b9364b75a3a7b522f3e2a5d6 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/33948893b9364b75a3a7b522f3e2a5d6 2024-11-07T12:42:14,014 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/33948893b9364b75a3a7b522f3e2a5d6, entries=150, sequenceid=161, filesize=11.9 K 2024-11-07T12:42:14,015 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/7ca0ade2a2f4482682d7474b13fb86a2 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/7ca0ade2a2f4482682d7474b13fb86a2 2024-11-07T12:42:14,030 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/7ca0ade2a2f4482682d7474b13fb86a2, entries=150, sequenceid=161, filesize=11.9 K 2024-11-07T12:42:14,031 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 8dc13886c19c8110f075e43282960321 in 625ms, sequenceid=161, compaction requested=true 2024-11-07T12:42:14,031 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8dc13886c19c8110f075e43282960321: 2024-11-07T12:42:14,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8dc13886c19c8110f075e43282960321:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:42:14,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:42:14,031 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:42:14,031 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:42:14,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8dc13886c19c8110f075e43282960321:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:42:14,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:42:14,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8dc13886c19c8110f075e43282960321:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:42:14,032 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36611 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:42:14,033 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): 8dc13886c19c8110f075e43282960321/A is initiating minor compaction (all files) 2024-11-07T12:42:14,033 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8dc13886c19c8110f075e43282960321/A in TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:14,033 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/f4c22dffcd63417cba781ba4a7661ef6, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/028e09b4fe5346239a93a80b44fb8fea, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/3a9f823b96a044fe8612cb11c76313b2] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp, totalSize=35.8 K 2024-11-07T12:42:14,033 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36611 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:42:14,033 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 8dc13886c19c8110f075e43282960321/B is initiating minor compaction (all files) 2024-11-07T12:42:14,033 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8dc13886c19c8110f075e43282960321/B in TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:14,033 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/f1f9e3ca46574b6fba6bfd1e572040ea, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/b4ed6e537731444b8c90a0bf75f8773c, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/33948893b9364b75a3a7b522f3e2a5d6] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp, totalSize=35.8 K 2024-11-07T12:42:14,035 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting f1f9e3ca46574b6fba6bfd1e572040ea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1730983330895 2024-11-07T12:42:14,035 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4c22dffcd63417cba781ba4a7661ef6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1730983330895 2024-11-07T12:42:14,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:42:14,036 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting b4ed6e537731444b8c90a0bf75f8773c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1730983331555 2024-11-07T12:42:14,036 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 33948893b9364b75a3a7b522f3e2a5d6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1730983332750 2024-11-07T12:42:14,036 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 028e09b4fe5346239a93a80b44fb8fea, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1730983331555 2024-11-07T12:42:14,037 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a9f823b96a044fe8612cb11c76313b2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1730983332750 2024-11-07T12:42:14,056 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8dc13886c19c8110f075e43282960321#A#compaction#231 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:42:14,057 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/c65833e71e824b6b833ea4f62a125fbc is 50, key is test_row_0/A:col10/1730983332750/Put/seqid=0 2024-11-07T12:42:14,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-07T12:42:14,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 8dc13886c19c8110f075e43282960321 2024-11-07T12:42:14,067 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8dc13886c19c8110f075e43282960321#B#compaction#232 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:42:14,067 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8dc13886c19c8110f075e43282960321 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-07T12:42:14,067 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/7ef02e120f4f48e8ac6be67260650987 is 50, key is test_row_0/B:col10/1730983332750/Put/seqid=0 2024-11-07T12:42:14,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=A 2024-11-07T12:42:14,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:14,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=B 2024-11-07T12:42:14,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:14,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=C 2024-11-07T12:42:14,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:14,072 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-07T12:42:14,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:14,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. as already flushing 2024-11-07T12:42:14,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:14,072 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:14,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:14,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:14,086 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/02be950e09bd4f30b7529f46f9dbb613 is 50, key is test_row_0/A:col10/1730983334065/Put/seqid=0 2024-11-07T12:42:14,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742105_1281 (size=12561) 2024-11-07T12:42:14,111 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/c65833e71e824b6b833ea4f62a125fbc as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/c65833e71e824b6b833ea4f62a125fbc 2024-11-07T12:42:14,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:14,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42034 deadline: 1730983394104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:14,116 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:14,116 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:14,116 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:14,119 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8dc13886c19c8110f075e43282960321/A of 8dc13886c19c8110f075e43282960321 into c65833e71e824b6b833ea4f62a125fbc(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:42:14,119 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8dc13886c19c8110f075e43282960321: 2024-11-07T12:42:14,119 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., storeName=8dc13886c19c8110f075e43282960321/A, priority=13, startTime=1730983334031; duration=0sec 2024-11-07T12:42:14,119 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:42:14,119 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8dc13886c19c8110f075e43282960321:A 2024-11-07T12:42:14,119 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:14,119 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:42:14,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41976 deadline: 1730983394107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:14,119 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:14,119 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:14,119 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:14,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:14,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42070 deadline: 1730983394109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:14,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:14,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:14,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:14,120 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36611 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:42:14,120 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): 8dc13886c19c8110f075e43282960321/C is initiating minor compaction (all files) 2024-11-07T12:42:14,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:14,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42026 deadline: 1730983394110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:14,121 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8dc13886c19c8110f075e43282960321/C in TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:14,121 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/0d2ec7477807410385f2f33995e21377, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/d2b18571482a46be9d0cb6c95864a1ec, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/7ca0ade2a2f4482682d7474b13fb86a2] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp, totalSize=35.8 K 2024-11-07T12:42:14,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:14,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:14,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:14,121 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d2ec7477807410385f2f33995e21377, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1730983330895 2024-11-07T12:42:14,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:14,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42054 deadline: 1730983394112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:14,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:14,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:14,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:14,122 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2b18571482a46be9d0cb6c95864a1ec, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1730983331555 2024-11-07T12:42:14,122 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ca0ade2a2f4482682d7474b13fb86a2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1730983332750 2024-11-07T12:42:14,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742106_1282 (size=12561) 2024-11-07T12:42:14,142 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/7ef02e120f4f48e8ac6be67260650987 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/7ef02e120f4f48e8ac6be67260650987 2024-11-07T12:42:14,148 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8dc13886c19c8110f075e43282960321/B of 8dc13886c19c8110f075e43282960321 into 7ef02e120f4f48e8ac6be67260650987(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:42:14,148 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8dc13886c19c8110f075e43282960321: 2024-11-07T12:42:14,148 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., storeName=8dc13886c19c8110f075e43282960321/B, priority=13, startTime=1730983334031; duration=0sec 2024-11-07T12:42:14,148 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:42:14,148 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8dc13886c19c8110f075e43282960321:B 2024-11-07T12:42:14,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742107_1283 (size=14541) 2024-11-07T12:42:14,153 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8dc13886c19c8110f075e43282960321#C#compaction#234 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:42:14,153 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/02be950e09bd4f30b7529f46f9dbb613 2024-11-07T12:42:14,153 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/6a5809c7514e4b2a9a47a631fca8a662 is 50, key is test_row_0/C:col10/1730983332750/Put/seqid=0 2024-11-07T12:42:14,171 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/2a391260d5344a6399f6486b3f7f9fc1 is 50, key is test_row_0/B:col10/1730983334065/Put/seqid=0 2024-11-07T12:42:14,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742108_1284 (size=12561) 2024-11-07T12:42:14,199 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/6a5809c7514e4b2a9a47a631fca8a662 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/6a5809c7514e4b2a9a47a631fca8a662 2024-11-07T12:42:14,206 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8dc13886c19c8110f075e43282960321/C of 8dc13886c19c8110f075e43282960321 into 6a5809c7514e4b2a9a47a631fca8a662(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:42:14,206 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8dc13886c19c8110f075e43282960321: 2024-11-07T12:42:14,206 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., storeName=8dc13886c19c8110f075e43282960321/C, priority=13, startTime=1730983334032; duration=0sec 2024-11-07T12:42:14,206 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:42:14,206 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8dc13886c19c8110f075e43282960321:C 2024-11-07T12:42:14,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742109_1285 (size=12151) 2024-11-07T12:42:14,211 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/2a391260d5344a6399f6486b3f7f9fc1 2024-11-07T12:42:14,220 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/a43576dd9ea44aa5bc9de424cfa96e23 is 50, key is test_row_0/C:col10/1730983334065/Put/seqid=0 2024-11-07T12:42:14,225 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:14,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42034 deadline: 1730983394222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:14,225 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:14,225 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:14,225 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:14,225 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:14,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41976 deadline: 1730983394222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:14,226 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:14,226 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:14,226 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:14,227 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-07T12:42:14,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:14,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. as already flushing 2024-11-07T12:42:14,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:14,227 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:14,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:14,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:14,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:14,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42026 deadline: 1730983394233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:14,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:14,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42054 deadline: 1730983394234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:14,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:14,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:14,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:14,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:14,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42070 deadline: 1730983394235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:14,240 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:14,240 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:14,240 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:14,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742110_1286 (size=12151) 2024-11-07T12:42:14,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:14,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:14,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:14,249 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/a43576dd9ea44aa5bc9de424cfa96e23 2024-11-07T12:42:14,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/02be950e09bd4f30b7529f46f9dbb613 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/02be950e09bd4f30b7529f46f9dbb613 2024-11-07T12:42:14,263 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/02be950e09bd4f30b7529f46f9dbb613, entries=200, sequenceid=174, filesize=14.2 K 2024-11-07T12:42:14,264 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/2a391260d5344a6399f6486b3f7f9fc1 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/2a391260d5344a6399f6486b3f7f9fc1 2024-11-07T12:42:14,270 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/2a391260d5344a6399f6486b3f7f9fc1, entries=150, sequenceid=174, filesize=11.9 K 2024-11-07T12:42:14,273 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/a43576dd9ea44aa5bc9de424cfa96e23 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/a43576dd9ea44aa5bc9de424cfa96e23 2024-11-07T12:42:14,279 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/a43576dd9ea44aa5bc9de424cfa96e23, entries=150, sequenceid=174, filesize=11.9 K 2024-11-07T12:42:14,280 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 8dc13886c19c8110f075e43282960321 in 213ms, sequenceid=174, compaction requested=false 2024-11-07T12:42:14,280 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8dc13886c19c8110f075e43282960321: 2024-11-07T12:42:14,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-07T12:42:14,385 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-07T12:42:14,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:14,386 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 8dc13886c19c8110f075e43282960321 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-07T12:42:14,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=A 2024-11-07T12:42:14,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:14,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=B 2024-11-07T12:42:14,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:14,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=C 2024-11-07T12:42:14,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:14,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/6a23d80af0084823afa4d4d2ef294d4b is 50, key is test_row_0/A:col10/1730983334109/Put/seqid=0 2024-11-07T12:42:14,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742111_1287 (size=12151) 2024-11-07T12:42:14,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. as already flushing 2024-11-07T12:42:14,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 8dc13886c19c8110f075e43282960321 2024-11-07T12:42:14,495 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/6a23d80af0084823afa4d4d2ef294d4b 2024-11-07T12:42:14,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/e0d27685b06e49169f4ca7ce1878caa3 is 50, key is test_row_0/B:col10/1730983334109/Put/seqid=0 2024-11-07T12:42:14,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:14,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42026 deadline: 1730983394517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:14,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:14,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42034 deadline: 1730983394520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:14,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:14,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41976 deadline: 1730983394520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:14,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:14,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42054 deadline: 1730983394521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:14,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:14,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:14,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:14,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:14,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42070 deadline: 1730983394525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:14,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:14,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:14,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:14,532 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:14,532 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:14,532 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:14,532 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:14,532 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:14,532 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:14,532 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:14,532 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:14,532 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:14,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742112_1288 (size=12151) 2024-11-07T12:42:14,569 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/e0d27685b06e49169f4ca7ce1878caa3 2024-11-07T12:42:14,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/6a343ced76af4ca6aaa9cbd4b60c28d2 is 50, key is test_row_0/C:col10/1730983334109/Put/seqid=0 2024-11-07T12:42:14,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742113_1289 (size=12151) 2024-11-07T12:42:14,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:14,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42054 deadline: 1730983394642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:14,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:14,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:14,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:14,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:14,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:14,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42034 deadline: 1730983394642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:14,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41976 deadline: 1730983394643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:14,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:14,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:14,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:14,646 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:14,646 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:14,646 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:14,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:14,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42070 deadline: 1730983394645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:14,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:14,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:14,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:14,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42026 deadline: 1730983394649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:14,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:14,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:14,661 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:14,661 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:14,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:14,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42034 deadline: 1730983394851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:14,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:14,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:14,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:14,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42054 deadline: 1730983394851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:14,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:14,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:14,855 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:14,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41976 deadline: 1730983394852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:14,855 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:14,855 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:14,865 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:14,865 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:14,865 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:14,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:14,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42070 deadline: 1730983394862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:14,866 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:14,866 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:14,866 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:14,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:14,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42026 deadline: 1730983394872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:14,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:14,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:14,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:14,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-07T12:42:15,037 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/6a343ced76af4ca6aaa9cbd4b60c28d2 2024-11-07T12:42:15,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/6a23d80af0084823afa4d4d2ef294d4b as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/6a23d80af0084823afa4d4d2ef294d4b 2024-11-07T12:42:15,051 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/6a23d80af0084823afa4d4d2ef294d4b, entries=150, sequenceid=196, filesize=11.9 K 2024-11-07T12:42:15,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/e0d27685b06e49169f4ca7ce1878caa3 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/e0d27685b06e49169f4ca7ce1878caa3 2024-11-07T12:42:15,063 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/e0d27685b06e49169f4ca7ce1878caa3, entries=150, sequenceid=196, filesize=11.9 K 2024-11-07T12:42:15,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/6a343ced76af4ca6aaa9cbd4b60c28d2 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/6a343ced76af4ca6aaa9cbd4b60c28d2 2024-11-07T12:42:15,071 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/6a343ced76af4ca6aaa9cbd4b60c28d2, entries=150, sequenceid=196, filesize=11.9 K 2024-11-07T12:42:15,072 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 8dc13886c19c8110f075e43282960321 in 687ms, sequenceid=196, compaction requested=true 2024-11-07T12:42:15,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 8dc13886c19c8110f075e43282960321: 2024-11-07T12:42:15,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:15,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-07T12:42:15,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-07T12:42:15,077 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-07T12:42:15,078 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3210 sec 2024-11-07T12:42:15,080 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.3450 sec 2024-11-07T12:42:15,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 8dc13886c19c8110f075e43282960321 2024-11-07T12:42:15,165 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8dc13886c19c8110f075e43282960321 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-07T12:42:15,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=A 2024-11-07T12:42:15,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:15,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=B 2024-11-07T12:42:15,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:15,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=C 2024-11-07T12:42:15,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:15,171 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/557f0816c56c43158adc93f3dd02e23e is 50, key is test_row_0/A:col10/1730983334482/Put/seqid=0 2024-11-07T12:42:15,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742114_1290 (size=14541) 2024-11-07T12:42:15,191 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/557f0816c56c43158adc93f3dd02e23e 2024-11-07T12:42:15,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:15,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42034 deadline: 1730983395191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:15,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:15,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42026 deadline: 1730983395194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:15,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:15,200 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:15,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42070 deadline: 1730983395194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:15,200 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:15,200 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:15,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:15,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:15,201 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:15,205 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:15,205 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:15,205 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:15,206 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/25451cf92b4344738e84fa89ea662041 is 50, key is test_row_0/B:col10/1730983334482/Put/seqid=0 2024-11-07T12:42:15,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:15,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41976 deadline: 1730983395198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:15,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:15,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42054 deadline: 1730983395207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:15,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:15,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:15,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:15,216 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:15,217 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:15,217 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:15,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742115_1291 (size=12151) 2024-11-07T12:42:15,220 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/25451cf92b4344738e84fa89ea662041 2024-11-07T12:42:15,250 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/737af3c578744549ad552f898f250fcf is 50, key is test_row_0/C:col10/1730983334482/Put/seqid=0 2024-11-07T12:42:15,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742116_1292 (size=12151) 2024-11-07T12:42:15,287 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/737af3c578744549ad552f898f250fcf 2024-11-07T12:42:15,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/557f0816c56c43158adc93f3dd02e23e as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/557f0816c56c43158adc93f3dd02e23e 2024-11-07T12:42:15,300 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/557f0816c56c43158adc93f3dd02e23e, entries=200, sequenceid=210, filesize=14.2 K 2024-11-07T12:42:15,301 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/25451cf92b4344738e84fa89ea662041 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/25451cf92b4344738e84fa89ea662041 2024-11-07T12:42:15,305 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/25451cf92b4344738e84fa89ea662041, entries=150, sequenceid=210, filesize=11.9 K 2024-11-07T12:42:15,306 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/737af3c578744549ad552f898f250fcf as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/737af3c578744549ad552f898f250fcf 2024-11-07T12:42:15,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:15,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42034 deadline: 1730983395312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:15,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:15,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:15,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:15,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42070 deadline: 1730983395312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:15,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:15,315 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:15,315 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:15,315 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:15,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:15,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42026 deadline: 1730983395312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:15,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:15,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:15,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:15,320 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/737af3c578744549ad552f898f250fcf, entries=150, sequenceid=210, filesize=11.9 K 2024-11-07T12:42:15,321 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 8dc13886c19c8110f075e43282960321 in 156ms, sequenceid=210, compaction requested=true 2024-11-07T12:42:15,321 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8dc13886c19c8110f075e43282960321: 2024-11-07T12:42:15,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8dc13886c19c8110f075e43282960321:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:42:15,321 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:42:15,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:42:15,321 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:42:15,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8dc13886c19c8110f075e43282960321:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:42:15,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:42:15,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8dc13886c19c8110f075e43282960321:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:42:15,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:42:15,323 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53794 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:42:15,323 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): 8dc13886c19c8110f075e43282960321/A is initiating minor compaction (all files) 2024-11-07T12:42:15,323 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8dc13886c19c8110f075e43282960321/A in TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:15,323 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/c65833e71e824b6b833ea4f62a125fbc, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/02be950e09bd4f30b7529f46f9dbb613, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/6a23d80af0084823afa4d4d2ef294d4b, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/557f0816c56c43158adc93f3dd02e23e] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp, totalSize=52.5 K 2024-11-07T12:42:15,324 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:42:15,324 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 8dc13886c19c8110f075e43282960321/B is initiating minor compaction (all files) 2024-11-07T12:42:15,324 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8dc13886c19c8110f075e43282960321/B in TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:15,324 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/7ef02e120f4f48e8ac6be67260650987, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/2a391260d5344a6399f6486b3f7f9fc1, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/e0d27685b06e49169f4ca7ce1878caa3, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/25451cf92b4344738e84fa89ea662041] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp, totalSize=47.9 K 2024-11-07T12:42:15,325 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ef02e120f4f48e8ac6be67260650987, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1730983332750 2024-11-07T12:42:15,325 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting c65833e71e824b6b833ea4f62a125fbc, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1730983332750 2024-11-07T12:42:15,325 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a391260d5344a6399f6486b3f7f9fc1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1730983333426 2024-11-07T12:42:15,325 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 02be950e09bd4f30b7529f46f9dbb613, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1730983333426 2024-11-07T12:42:15,326 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting e0d27685b06e49169f4ca7ce1878caa3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1730983334093 2024-11-07T12:42:15,326 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a23d80af0084823afa4d4d2ef294d4b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1730983334093 2024-11-07T12:42:15,326 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 25451cf92b4344738e84fa89ea662041, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1730983334482 2024-11-07T12:42:15,326 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 557f0816c56c43158adc93f3dd02e23e, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1730983334482 2024-11-07T12:42:15,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 8dc13886c19c8110f075e43282960321 2024-11-07T12:42:15,328 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8dc13886c19c8110f075e43282960321 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-07T12:42:15,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=A 2024-11-07T12:42:15,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:15,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=B 2024-11-07T12:42:15,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:15,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=C 2024-11-07T12:42:15,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:15,343 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/a3a9d59ba7d649cbaa6fb98c1626fdbc is 50, key is test_row_0/A:col10/1730983335325/Put/seqid=0 2024-11-07T12:42:15,346 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8dc13886c19c8110f075e43282960321#A#compaction#244 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:42:15,347 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/09d00082c9ae41a49edb4f1e9dfc399f is 50, key is test_row_0/A:col10/1730983334482/Put/seqid=0 2024-11-07T12:42:15,350 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8dc13886c19c8110f075e43282960321#B#compaction#245 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:42:15,351 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/b326d395a8c54cb6a850ed756cb8a9ba is 50, key is test_row_0/B:col10/1730983334482/Put/seqid=0 2024-11-07T12:42:15,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742117_1293 (size=14541) 2024-11-07T12:42:15,364 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/a3a9d59ba7d649cbaa6fb98c1626fdbc 2024-11-07T12:42:15,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:15,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41976 deadline: 1730983395357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:15,366 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:15,366 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:15,366 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:15,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:15,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42054 deadline: 1730983395364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:15,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:15,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:15,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:15,401 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/03c2ea01c3d54a3c858ce41cabaffaf0 is 50, key is test_row_0/B:col10/1730983335325/Put/seqid=0 2024-11-07T12:42:15,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742118_1294 (size=12697) 2024-11-07T12:42:15,418 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/09d00082c9ae41a49edb4f1e9dfc399f as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/09d00082c9ae41a49edb4f1e9dfc399f 2024-11-07T12:42:15,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742120_1296 (size=12151) 2024-11-07T12:42:15,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742119_1295 (size=12697) 2024-11-07T12:42:15,422 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/03c2ea01c3d54a3c858ce41cabaffaf0 2024-11-07T12:42:15,430 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8dc13886c19c8110f075e43282960321/A of 8dc13886c19c8110f075e43282960321 into 09d00082c9ae41a49edb4f1e9dfc399f(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:42:15,430 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8dc13886c19c8110f075e43282960321: 2024-11-07T12:42:15,430 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., storeName=8dc13886c19c8110f075e43282960321/A, priority=12, startTime=1730983335321; duration=0sec 2024-11-07T12:42:15,431 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/b326d395a8c54cb6a850ed756cb8a9ba as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/b326d395a8c54cb6a850ed756cb8a9ba 2024-11-07T12:42:15,431 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:42:15,431 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8dc13886c19c8110f075e43282960321:A 2024-11-07T12:42:15,431 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:42:15,437 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/e215b50ea1b343078b8865f6a7207db8 is 50, key is test_row_0/C:col10/1730983335325/Put/seqid=0 2024-11-07T12:42:15,437 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:42:15,437 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): 8dc13886c19c8110f075e43282960321/C is initiating minor compaction (all files) 2024-11-07T12:42:15,437 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8dc13886c19c8110f075e43282960321/C in TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:15,437 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/6a5809c7514e4b2a9a47a631fca8a662, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/a43576dd9ea44aa5bc9de424cfa96e23, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/6a343ced76af4ca6aaa9cbd4b60c28d2, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/737af3c578744549ad552f898f250fcf] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp, totalSize=47.9 K 2024-11-07T12:42:15,438 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a5809c7514e4b2a9a47a631fca8a662, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1730983332750 2024-11-07T12:42:15,438 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting a43576dd9ea44aa5bc9de424cfa96e23, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1730983333426 2024-11-07T12:42:15,439 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a343ced76af4ca6aaa9cbd4b60c28d2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1730983334093 2024-11-07T12:42:15,439 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 737af3c578744549ad552f898f250fcf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1730983334482 2024-11-07T12:42:15,442 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8dc13886c19c8110f075e43282960321/B of 8dc13886c19c8110f075e43282960321 into b326d395a8c54cb6a850ed756cb8a9ba(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:42:15,442 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8dc13886c19c8110f075e43282960321: 2024-11-07T12:42:15,442 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., storeName=8dc13886c19c8110f075e43282960321/B, priority=12, startTime=1730983335321; duration=0sec 2024-11-07T12:42:15,443 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:42:15,443 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8dc13886c19c8110f075e43282960321:B 2024-11-07T12:42:15,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742121_1297 (size=12151) 2024-11-07T12:42:15,468 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/e215b50ea1b343078b8865f6a7207db8 2024-11-07T12:42:15,475 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/a3a9d59ba7d649cbaa6fb98c1626fdbc as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/a3a9d59ba7d649cbaa6fb98c1626fdbc 2024-11-07T12:42:15,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:15,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41976 deadline: 1730983395472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:15,476 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:15,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:15,476 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:15,476 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:15,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42054 deadline: 1730983395472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:15,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:15,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:15,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:15,481 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8dc13886c19c8110f075e43282960321#C#compaction#248 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:42:15,481 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/a3a9d59ba7d649cbaa6fb98c1626fdbc, entries=200, sequenceid=230, filesize=14.2 K 2024-11-07T12:42:15,482 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/20dd8b947f5a40c8932bbdfb56e66849 is 50, key is test_row_0/C:col10/1730983334482/Put/seqid=0 2024-11-07T12:42:15,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/03c2ea01c3d54a3c858ce41cabaffaf0 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/03c2ea01c3d54a3c858ce41cabaffaf0 2024-11-07T12:42:15,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742122_1298 (size=12697) 2024-11-07T12:42:15,491 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/03c2ea01c3d54a3c858ce41cabaffaf0, entries=150, sequenceid=230, filesize=11.9 K 2024-11-07T12:42:15,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/e215b50ea1b343078b8865f6a7207db8 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/e215b50ea1b343078b8865f6a7207db8 2024-11-07T12:42:15,501 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/e215b50ea1b343078b8865f6a7207db8, entries=150, sequenceid=230, filesize=11.9 K 2024-11-07T12:42:15,502 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 8dc13886c19c8110f075e43282960321 in 175ms, sequenceid=230, compaction requested=false 2024-11-07T12:42:15,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8dc13886c19c8110f075e43282960321: 2024-11-07T12:42:15,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 8dc13886c19c8110f075e43282960321 2024-11-07T12:42:15,524 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8dc13886c19c8110f075e43282960321 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-07T12:42:15,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=A 2024-11-07T12:42:15,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:15,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=B 2024-11-07T12:42:15,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:15,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=C 2024-11-07T12:42:15,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:15,540 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/9ed670d3d0a24dbb9cd78f7450bda788 is 50, key is test_row_0/A:col10/1730983335348/Put/seqid=0 2024-11-07T12:42:15,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:15,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42070 deadline: 1730983395563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:15,572 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:15,572 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:15,572 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:15,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742123_1299 (size=14541) 2024-11-07T12:42:15,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:15,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42034 deadline: 1730983395569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:15,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:15,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42026 deadline: 1730983395569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:15,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:15,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:15,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:15,574 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/9ed670d3d0a24dbb9cd78f7450bda788 2024-11-07T12:42:15,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:15,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:15,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:15,598 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/2459f784c8cd48f9a009974b7dade643 is 50, key is test_row_0/B:col10/1730983335348/Put/seqid=0 2024-11-07T12:42:15,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742124_1300 (size=12151) 2024-11-07T12:42:15,649 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/2459f784c8cd48f9a009974b7dade643 2024-11-07T12:42:15,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:15,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42054 deadline: 1730983395682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:15,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:15,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:15,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:15,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:15,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41976 deadline: 1730983395682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:15,688 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:15,688 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:15,688 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:15,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:15,690 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/b0a7cc5219394fa48f248e0a9ac8fd44 is 50, key is test_row_0/C:col10/1730983335348/Put/seqid=0 2024-11-07T12:42:15,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:15,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42070 deadline: 1730983395684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:15,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:15,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42034 deadline: 1730983395684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:15,691 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:15,692 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:15,692 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:15,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42026 deadline: 1730983395683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:15,692 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:15,692 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:15,693 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:15,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:15,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:15,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:15,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742125_1301 (size=12151) 2024-11-07T12:42:15,732 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/b0a7cc5219394fa48f248e0a9ac8fd44 2024-11-07T12:42:15,770 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/9ed670d3d0a24dbb9cd78f7450bda788 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/9ed670d3d0a24dbb9cd78f7450bda788 2024-11-07T12:42:15,806 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/9ed670d3d0a24dbb9cd78f7450bda788, entries=200, sequenceid=246, filesize=14.2 K 2024-11-07T12:42:15,808 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/2459f784c8cd48f9a009974b7dade643 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/2459f784c8cd48f9a009974b7dade643 2024-11-07T12:42:15,834 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/2459f784c8cd48f9a009974b7dade643, entries=150, sequenceid=246, filesize=11.9 K 2024-11-07T12:42:15,845 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/b0a7cc5219394fa48f248e0a9ac8fd44 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/b0a7cc5219394fa48f248e0a9ac8fd44 2024-11-07T12:42:15,883 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/b0a7cc5219394fa48f248e0a9ac8fd44, entries=150, sequenceid=246, filesize=11.9 K 2024-11-07T12:42:15,884 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 8dc13886c19c8110f075e43282960321 in 360ms, sequenceid=246, compaction requested=true 2024-11-07T12:42:15,884 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8dc13886c19c8110f075e43282960321: 2024-11-07T12:42:15,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8dc13886c19c8110f075e43282960321:A, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:42:15,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:42:15,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8dc13886c19c8110f075e43282960321:B, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:42:15,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:42:15,885 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:42:15,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8dc13886c19c8110f075e43282960321:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:42:15,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-07T12:42:15,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-07T12:42:15,892 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:42:15,892 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:42:15,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-07T12:42:15,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-07T12:42:15,897 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41779 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:42:15,897 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 8dc13886c19c8110f075e43282960321/A is initiating minor compaction (all files) 2024-11-07T12:42:15,897 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8dc13886c19c8110f075e43282960321/A in TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:15,897 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/09d00082c9ae41a49edb4f1e9dfc399f, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/a3a9d59ba7d649cbaa6fb98c1626fdbc, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/9ed670d3d0a24dbb9cd78f7450bda788] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp, totalSize=40.8 K 2024-11-07T12:42:15,899 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:42:15,899 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 09d00082c9ae41a49edb4f1e9dfc399f, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1730983334482 2024-11-07T12:42:15,900 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting a3a9d59ba7d649cbaa6fb98c1626fdbc, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1730983335187 2024-11-07T12:42:15,900 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ed670d3d0a24dbb9cd78f7450bda788, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1730983335348 2024-11-07T12:42:15,900 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:42:15,900 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:42:15,907 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8dc13886c19c8110f075e43282960321 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-07T12:42:15,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=A 2024-11-07T12:42:15,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:15,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=B 2024-11-07T12:42:15,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:15,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=C 2024-11-07T12:42:15,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:15,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 8dc13886c19c8110f075e43282960321 2024-11-07T12:42:15,916 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/316944d61b8d4e2ab5c7e585b3e0ebae is 50, key is test_row_0/A:col10/1730983335563/Put/seqid=0 2024-11-07T12:42:15,917 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/20dd8b947f5a40c8932bbdfb56e66849 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/20dd8b947f5a40c8932bbdfb56e66849 2024-11-07T12:42:15,923 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8dc13886c19c8110f075e43282960321#A#compaction#253 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:42:15,924 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/5f0be8de0746483cad0da1ce0ccc4671 is 50, key is test_row_0/A:col10/1730983335348/Put/seqid=0 2024-11-07T12:42:15,927 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8dc13886c19c8110f075e43282960321/C of 8dc13886c19c8110f075e43282960321 into 20dd8b947f5a40c8932bbdfb56e66849(size=12.4 K), total size for store is 36.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:42:15,927 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8dc13886c19c8110f075e43282960321: 2024-11-07T12:42:15,927 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., storeName=8dc13886c19c8110f075e43282960321/C, priority=12, startTime=1730983335322; duration=0sec 2024-11-07T12:42:15,927 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-07T12:42:15,927 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8dc13886c19c8110f075e43282960321:C 2024-11-07T12:42:15,927 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8dc13886c19c8110f075e43282960321:C 2024-11-07T12:42:15,927 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:42:15,929 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36999 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:42:15,929 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): 8dc13886c19c8110f075e43282960321/B is initiating minor compaction (all files) 2024-11-07T12:42:15,931 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8dc13886c19c8110f075e43282960321/B in TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:15,931 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/b326d395a8c54cb6a850ed756cb8a9ba, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/03c2ea01c3d54a3c858ce41cabaffaf0, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/2459f784c8cd48f9a009974b7dade643] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp, totalSize=36.1 K 2024-11-07T12:42:15,931 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting b326d395a8c54cb6a850ed756cb8a9ba, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1730983334482 2024-11-07T12:42:15,932 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03c2ea01c3d54a3c858ce41cabaffaf0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1730983335187 2024-11-07T12:42:15,932 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2459f784c8cd48f9a009974b7dade643, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1730983335348 2024-11-07T12:42:15,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:15,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42070 deadline: 1730983395932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:15,942 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:15,942 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:15,942 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:15,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:15,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42034 deadline: 1730983395938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:15,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:15,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:15,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:15,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:15,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42026 deadline: 1730983395942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:15,948 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:15,948 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:15,948 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:15,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742126_1302 (size=12251) 2024-11-07T12:42:15,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742127_1303 (size=12799) 2024-11-07T12:42:15,985 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8dc13886c19c8110f075e43282960321#B#compaction#254 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:42:15,986 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/b6a059a0f7be47ada66f32acfe122fc5 is 50, key is test_row_0/B:col10/1730983335348/Put/seqid=0 2024-11-07T12:42:15,992 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/5f0be8de0746483cad0da1ce0ccc4671 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/5f0be8de0746483cad0da1ce0ccc4671 2024-11-07T12:42:15,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:15,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42054 deadline: 1730983395992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:15,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:15,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41976 deadline: 1730983395992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:15,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:15,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:15,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:15,995 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:15,995 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:15,996 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:16,000 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8dc13886c19c8110f075e43282960321/A of 8dc13886c19c8110f075e43282960321 into 5f0be8de0746483cad0da1ce0ccc4671(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:42:16,000 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8dc13886c19c8110f075e43282960321: 2024-11-07T12:42:16,000 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., storeName=8dc13886c19c8110f075e43282960321/A, priority=13, startTime=1730983335884; duration=0sec 2024-11-07T12:42:16,000 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:42:16,000 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8dc13886c19c8110f075e43282960321:A 2024-11-07T12:42:16,000 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:42:16,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-07T12:42:16,002 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36999 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:42:16,002 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 8dc13886c19c8110f075e43282960321/C is initiating minor compaction (all files) 2024-11-07T12:42:16,002 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8dc13886c19c8110f075e43282960321/C in TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:16,002 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/20dd8b947f5a40c8932bbdfb56e66849, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/e215b50ea1b343078b8865f6a7207db8, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/b0a7cc5219394fa48f248e0a9ac8fd44] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp, totalSize=36.1 K 2024-11-07T12:42:16,003 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 20dd8b947f5a40c8932bbdfb56e66849, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1730983334482 2024-11-07T12:42:16,003 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting e215b50ea1b343078b8865f6a7207db8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1730983335187 2024-11-07T12:42:16,003 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting b0a7cc5219394fa48f248e0a9ac8fd44, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1730983335348 2024-11-07T12:42:16,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742128_1304 (size=12799) 2024-11-07T12:42:16,033 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8dc13886c19c8110f075e43282960321#C#compaction#255 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:42:16,033 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/6e3e1f2415dc461aaad8b7cc306f2c82 is 50, key is test_row_0/C:col10/1730983335348/Put/seqid=0 2024-11-07T12:42:16,056 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-07T12:42:16,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:16,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:16,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. as already flushing 2024-11-07T12:42:16,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42070 deadline: 1730983396052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:16,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:16,057 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:16,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:16,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:16,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:16,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:16,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:16,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:16,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42034 deadline: 1730983396053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:16,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:16,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:16,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:16,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:16,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42026 deadline: 1730983396053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:16,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:16,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:16,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:16,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742129_1305 (size=12799) 2024-11-07T12:42:16,092 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/6e3e1f2415dc461aaad8b7cc306f2c82 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/6e3e1f2415dc461aaad8b7cc306f2c82 2024-11-07T12:42:16,100 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8dc13886c19c8110f075e43282960321/C of 8dc13886c19c8110f075e43282960321 into 6e3e1f2415dc461aaad8b7cc306f2c82(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:42:16,100 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8dc13886c19c8110f075e43282960321: 2024-11-07T12:42:16,100 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., storeName=8dc13886c19c8110f075e43282960321/C, priority=13, startTime=1730983335885; duration=0sec 2024-11-07T12:42:16,100 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:42:16,100 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8dc13886c19c8110f075e43282960321:C 2024-11-07T12:42:16,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-07T12:42:16,213 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-07T12:42:16,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:16,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. as already flushing 2024-11-07T12:42:16,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:16,214 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:16,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:16,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:16,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:16,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42070 deadline: 1730983396263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:16,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:16,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:16,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:16,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:16,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42034 deadline: 1730983396273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:16,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:16,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42026 deadline: 1730983396273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:16,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:16,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:16,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:16,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:16,281 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:16,281 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:16,370 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-07T12:42:16,375 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/316944d61b8d4e2ab5c7e585b3e0ebae 2024-11-07T12:42:16,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:16,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. as already flushing 2024-11-07T12:42:16,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:16,376 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:16,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:16,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:16,399 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/29deb66790584f99b4a69d0c19e03572 is 50, key is test_row_0/B:col10/1730983335563/Put/seqid=0 2024-11-07T12:42:16,429 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/b6a059a0f7be47ada66f32acfe122fc5 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/b6a059a0f7be47ada66f32acfe122fc5 2024-11-07T12:42:16,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742130_1306 (size=12251) 2024-11-07T12:42:16,446 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/29deb66790584f99b4a69d0c19e03572 2024-11-07T12:42:16,450 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8dc13886c19c8110f075e43282960321/B of 8dc13886c19c8110f075e43282960321 into b6a059a0f7be47ada66f32acfe122fc5(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:42:16,450 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8dc13886c19c8110f075e43282960321: 2024-11-07T12:42:16,450 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., storeName=8dc13886c19c8110f075e43282960321/B, priority=13, startTime=1730983335885; duration=0sec 2024-11-07T12:42:16,450 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:42:16,452 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8dc13886c19c8110f075e43282960321:B 2024-11-07T12:42:16,455 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/2882c2e15b4a4c1ca8acf97018af6a03 is 50, key is test_row_0/C:col10/1730983335563/Put/seqid=0 2024-11-07T12:42:16,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742131_1307 (size=12251) 2024-11-07T12:42:16,497 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/2882c2e15b4a4c1ca8acf97018af6a03 2024-11-07T12:42:16,505 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/316944d61b8d4e2ab5c7e585b3e0ebae as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/316944d61b8d4e2ab5c7e585b3e0ebae 2024-11-07T12:42:16,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:16,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41976 deadline: 1730983396502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:16,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:16,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42054 deadline: 1730983396502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:16,506 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:16,506 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:16,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:16,507 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:16,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:16,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:16,511 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/316944d61b8d4e2ab5c7e585b3e0ebae, entries=150, sequenceid=265, filesize=12.0 K 2024-11-07T12:42:16,513 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/29deb66790584f99b4a69d0c19e03572 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/29deb66790584f99b4a69d0c19e03572 2024-11-07T12:42:16,524 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/29deb66790584f99b4a69d0c19e03572, entries=150, sequenceid=265, filesize=12.0 K 2024-11-07T12:42:16,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-07T12:42:16,525 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/2882c2e15b4a4c1ca8acf97018af6a03 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/2882c2e15b4a4c1ca8acf97018af6a03 2024-11-07T12:42:16,531 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/2882c2e15b4a4c1ca8acf97018af6a03, entries=150, sequenceid=265, filesize=12.0 K 2024-11-07T12:42:16,531 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-07T12:42:16,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:16,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. as already flushing 2024-11-07T12:42:16,532 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 8dc13886c19c8110f075e43282960321 in 625ms, sequenceid=265, compaction requested=false 2024-11-07T12:42:16,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:16,532 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8dc13886c19c8110f075e43282960321: 2024-11-07T12:42:16,532 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:16,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:16,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:16,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 8dc13886c19c8110f075e43282960321 2024-11-07T12:42:16,575 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8dc13886c19c8110f075e43282960321 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-07T12:42:16,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=A 2024-11-07T12:42:16,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:16,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=B 2024-11-07T12:42:16,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:16,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=C 2024-11-07T12:42:16,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:16,590 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/3015daabefb54d8295ddc554ff3750b7 is 50, key is test_row_0/A:col10/1730983336573/Put/seqid=0 2024-11-07T12:42:16,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742132_1308 (size=14741) 2024-11-07T12:42:16,604 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/3015daabefb54d8295ddc554ff3750b7 2024-11-07T12:42:16,613 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/1e5298171c6f4d17b79939bd0d1d3a4a is 50, key is test_row_0/B:col10/1730983336573/Put/seqid=0 2024-11-07T12:42:16,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742133_1309 (size=12301) 2024-11-07T12:42:16,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:16,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42026 deadline: 1730983396672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:16,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:16,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:16,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:16,686 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-07T12:42:16,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:16,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:16,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. as already flushing 2024-11-07T12:42:16,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:16,686 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:16,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:16,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42034 deadline: 1730983396677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:16,687 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:16,687 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:16,687 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:16,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:16,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42070 deadline: 1730983396680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:16,688 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:16,688 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:16,688 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:16,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:16,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:16,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42026 deadline: 1730983396791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:16,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:16,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:16,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:16,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:16,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42034 deadline: 1730983396791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:16,796 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:16,797 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:16,797 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:16,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:16,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42070 deadline: 1730983396792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:16,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:16,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:16,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:16,848 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-07T12:42:16,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:16,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. as already flushing 2024-11-07T12:42:16,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:16,848 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:16,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:16,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:17,002 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-07T12:42:17,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:17,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. as already flushing 2024-11-07T12:42:17,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:17,003 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:17,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:17,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:17,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:17,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42026 deadline: 1730983397001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:17,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:17,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:17,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:17,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:17,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42034 deadline: 1730983397002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:17,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:17,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42070 deadline: 1730983397002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:17,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:17,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:17,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:17,006 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:17,006 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:17,006 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:17,032 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/1e5298171c6f4d17b79939bd0d1d3a4a 2024-11-07T12:42:17,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-07T12:42:17,053 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/cf5597b105b042aa97fad28d3c8dfa8e is 50, key is test_row_0/C:col10/1730983336573/Put/seqid=0 2024-11-07T12:42:17,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742134_1310 (size=12301) 2024-11-07T12:42:17,068 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/cf5597b105b042aa97fad28d3c8dfa8e 2024-11-07T12:42:17,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/3015daabefb54d8295ddc554ff3750b7 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/3015daabefb54d8295ddc554ff3750b7 2024-11-07T12:42:17,078 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/3015daabefb54d8295ddc554ff3750b7, entries=200, sequenceid=283, filesize=14.4 K 2024-11-07T12:42:17,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/1e5298171c6f4d17b79939bd0d1d3a4a as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/1e5298171c6f4d17b79939bd0d1d3a4a 2024-11-07T12:42:17,089 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/1e5298171c6f4d17b79939bd0d1d3a4a, entries=150, sequenceid=283, filesize=12.0 K 2024-11-07T12:42:17,090 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/cf5597b105b042aa97fad28d3c8dfa8e as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/cf5597b105b042aa97fad28d3c8dfa8e 2024-11-07T12:42:17,102 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/cf5597b105b042aa97fad28d3c8dfa8e, entries=150, sequenceid=283, filesize=12.0 K 2024-11-07T12:42:17,103 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 8dc13886c19c8110f075e43282960321 in 528ms, sequenceid=283, compaction requested=true 2024-11-07T12:42:17,103 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8dc13886c19c8110f075e43282960321: 2024-11-07T12:42:17,104 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:42:17,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8dc13886c19c8110f075e43282960321:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:42:17,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:42:17,104 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:42:17,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8dc13886c19c8110f075e43282960321:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:42:17,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:42:17,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8dc13886c19c8110f075e43282960321:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:42:17,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:42:17,105 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39791 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:42:17,105 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): 8dc13886c19c8110f075e43282960321/A is initiating minor compaction (all files) 2024-11-07T12:42:17,105 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8dc13886c19c8110f075e43282960321/A in TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:17,105 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/5f0be8de0746483cad0da1ce0ccc4671, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/316944d61b8d4e2ab5c7e585b3e0ebae, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/3015daabefb54d8295ddc554ff3750b7] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp, totalSize=38.9 K 2024-11-07T12:42:17,106 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37351 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:42:17,106 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 8dc13886c19c8110f075e43282960321/B is initiating minor compaction (all files) 2024-11-07T12:42:17,106 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8dc13886c19c8110f075e43282960321/B in TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:17,106 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/b6a059a0f7be47ada66f32acfe122fc5, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/29deb66790584f99b4a69d0c19e03572, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/1e5298171c6f4d17b79939bd0d1d3a4a] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp, totalSize=36.5 K 2024-11-07T12:42:17,107 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f0be8de0746483cad0da1ce0ccc4671, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1730983335348 2024-11-07T12:42:17,107 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting b6a059a0f7be47ada66f32acfe122fc5, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1730983335348 2024-11-07T12:42:17,107 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 316944d61b8d4e2ab5c7e585b3e0ebae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1730983335544 2024-11-07T12:42:17,107 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 29deb66790584f99b4a69d0c19e03572, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1730983335544 2024-11-07T12:42:17,107 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3015daabefb54d8295ddc554ff3750b7, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1730983335919 2024-11-07T12:42:17,108 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e5298171c6f4d17b79939bd0d1d3a4a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1730983335935 2024-11-07T12:42:17,120 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8dc13886c19c8110f075e43282960321#B#compaction#261 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:42:17,121 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/613372ea89ec4e6a962c0c708a56f19d is 50, key is test_row_0/B:col10/1730983336573/Put/seqid=0 2024-11-07T12:42:17,127 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8dc13886c19c8110f075e43282960321#A#compaction#262 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:42:17,127 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/c7ff1c1a9a6143fcba89c5bc815231d6 is 50, key is test_row_0/A:col10/1730983336573/Put/seqid=0 2024-11-07T12:42:17,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742136_1312 (size=13051) 2024-11-07T12:42:17,156 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-07T12:42:17,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:17,157 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 8dc13886c19c8110f075e43282960321 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-07T12:42:17,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=A 2024-11-07T12:42:17,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:17,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=B 2024-11-07T12:42:17,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:17,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=C 2024-11-07T12:42:17,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:17,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742135_1311 (size=13051) 2024-11-07T12:42:17,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/bb7bd17bf41e47b78f8f8ef571c5f6ab is 50, key is test_row_0/A:col10/1730983336588/Put/seqid=0 2024-11-07T12:42:17,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742137_1313 (size=12301) 2024-11-07T12:42:17,178 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/bb7bd17bf41e47b78f8f8ef571c5f6ab 2024-11-07T12:42:17,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/519c8d48c87b47ebb01bb2236fb344d5 is 50, key is test_row_0/B:col10/1730983336588/Put/seqid=0 2024-11-07T12:42:17,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742138_1314 (size=12301) 2024-11-07T12:42:17,322 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. as already flushing 2024-11-07T12:42:17,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 8dc13886c19c8110f075e43282960321 2024-11-07T12:42:17,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:17,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42070 deadline: 1730983397354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:17,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:17,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:17,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:17,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:17,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42034 deadline: 1730983397359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:17,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:17,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42026 deadline: 1730983397361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:17,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:17,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:17,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:17,366 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:17,366 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:17,366 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:17,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:17,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42034 deadline: 1730983397471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:17,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:17,474 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:17,474 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:17,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42070 deadline: 1730983397471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:17,474 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:17,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:17,475 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:17,475 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:17,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42026 deadline: 1730983397471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:17,475 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:17,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:17,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:17,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:17,524 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:17,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41976 deadline: 1730983397521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:17,525 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:17,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42054 deadline: 1730983397521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:17,525 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to db9ad1cb6cf9,34293,1730983260217 for 'test_row_0' in 8dc13886c19c8110f075e43282960321 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:42:17,526 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:17,526 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:17,526 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:17,526 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to db9ad1cb6cf9,34293,1730983260217 for 'test_row_1' in 8dc13886c19c8110f075e43282960321 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:42:17,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:17,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:17,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:17,561 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/c7ff1c1a9a6143fcba89c5bc815231d6 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/c7ff1c1a9a6143fcba89c5bc815231d6 2024-11-07T12:42:17,564 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/613372ea89ec4e6a962c0c708a56f19d as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/613372ea89ec4e6a962c0c708a56f19d 2024-11-07T12:42:17,567 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8dc13886c19c8110f075e43282960321/A of 8dc13886c19c8110f075e43282960321 into c7ff1c1a9a6143fcba89c5bc815231d6(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:42:17,567 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8dc13886c19c8110f075e43282960321: 2024-11-07T12:42:17,567 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., storeName=8dc13886c19c8110f075e43282960321/A, priority=13, startTime=1730983337104; duration=0sec 2024-11-07T12:42:17,567 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:42:17,567 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8dc13886c19c8110f075e43282960321:A 2024-11-07T12:42:17,567 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:42:17,569 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37351 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:42:17,570 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8dc13886c19c8110f075e43282960321/B of 8dc13886c19c8110f075e43282960321 into 613372ea89ec4e6a962c0c708a56f19d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:42:17,570 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8dc13886c19c8110f075e43282960321: 2024-11-07T12:42:17,570 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., storeName=8dc13886c19c8110f075e43282960321/B, priority=13, startTime=1730983337104; duration=0sec 2024-11-07T12:42:17,570 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:42:17,570 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8dc13886c19c8110f075e43282960321:B 2024-11-07T12:42:17,569 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): 8dc13886c19c8110f075e43282960321/C is initiating minor compaction (all files) 2024-11-07T12:42:17,570 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8dc13886c19c8110f075e43282960321/C in TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:17,571 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/6e3e1f2415dc461aaad8b7cc306f2c82, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/2882c2e15b4a4c1ca8acf97018af6a03, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/cf5597b105b042aa97fad28d3c8dfa8e] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp, totalSize=36.5 K 2024-11-07T12:42:17,571 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e3e1f2415dc461aaad8b7cc306f2c82, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1730983335348 2024-11-07T12:42:17,571 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2882c2e15b4a4c1ca8acf97018af6a03, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1730983335544 2024-11-07T12:42:17,572 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf5597b105b042aa97fad28d3c8dfa8e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1730983335935 2024-11-07T12:42:17,583 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8dc13886c19c8110f075e43282960321#C#compaction#265 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:42:17,583 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/5b12a32838434a30bccd3af318159dd0 is 50, key is test_row_0/C:col10/1730983336573/Put/seqid=0 2024-11-07T12:42:17,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742139_1315 (size=13051) 2024-11-07T12:42:17,600 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/5b12a32838434a30bccd3af318159dd0 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/5b12a32838434a30bccd3af318159dd0 2024-11-07T12:42:17,607 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8dc13886c19c8110f075e43282960321/C of 8dc13886c19c8110f075e43282960321 into 5b12a32838434a30bccd3af318159dd0(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:42:17,607 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8dc13886c19c8110f075e43282960321: 2024-11-07T12:42:17,607 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., storeName=8dc13886c19c8110f075e43282960321/C, priority=13, startTime=1730983337104; duration=0sec 2024-11-07T12:42:17,607 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:42:17,607 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8dc13886c19c8110f075e43282960321:C 2024-11-07T12:42:17,650 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/519c8d48c87b47ebb01bb2236fb344d5 2024-11-07T12:42:17,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/b71dd55b27304789aed52531e7846d9a is 50, key is test_row_0/C:col10/1730983336588/Put/seqid=0 2024-11-07T12:42:17,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742140_1316 (size=12301) 2024-11-07T12:42:17,664 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/b71dd55b27304789aed52531e7846d9a 2024-11-07T12:42:17,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/bb7bd17bf41e47b78f8f8ef571c5f6ab as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/bb7bd17bf41e47b78f8f8ef571c5f6ab 2024-11-07T12:42:17,681 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/bb7bd17bf41e47b78f8f8ef571c5f6ab, entries=150, sequenceid=301, filesize=12.0 K 2024-11-07T12:42:17,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/519c8d48c87b47ebb01bb2236fb344d5 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/519c8d48c87b47ebb01bb2236fb344d5 2024-11-07T12:42:17,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:17,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42034 deadline: 1730983397682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:17,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:17,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:17,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:17,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:17,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42070 deadline: 1730983397682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:17,689 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:17,689 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:17,689 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:17,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:17,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42026 deadline: 1730983397683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:17,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:17,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:17,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:17,692 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/519c8d48c87b47ebb01bb2236fb344d5, entries=150, sequenceid=301, filesize=12.0 K 2024-11-07T12:42:17,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/b71dd55b27304789aed52531e7846d9a as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/b71dd55b27304789aed52531e7846d9a 2024-11-07T12:42:17,702 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/b71dd55b27304789aed52531e7846d9a, entries=150, sequenceid=301, filesize=12.0 K 2024-11-07T12:42:17,713 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 8dc13886c19c8110f075e43282960321 in 556ms, sequenceid=301, compaction requested=false 2024-11-07T12:42:17,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 8dc13886c19c8110f075e43282960321: 2024-11-07T12:42:17,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:17,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-07T12:42:17,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-07T12:42:17,718 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-07T12:42:17,718 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8160 sec 2024-11-07T12:42:17,722 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.8280 sec 2024-11-07T12:42:18,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 8dc13886c19c8110f075e43282960321 2024-11-07T12:42:18,006 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8dc13886c19c8110f075e43282960321 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-07T12:42:18,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=A 2024-11-07T12:42:18,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:18,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=B 2024-11-07T12:42:18,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:18,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=C 2024-11-07T12:42:18,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:18,015 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/e0baccc0f1724da2bf253afb89193399 is 50, key is test_row_0/A:col10/1730983338005/Put/seqid=0 2024-11-07T12:42:18,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:18,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42034 deadline: 1730983398028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:18,033 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:18,033 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:18,033 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:18,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:18,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42026 deadline: 1730983398029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:18,036 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:18,036 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:18,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:18,036 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:18,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42070 deadline: 1730983398030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:18,036 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:18,036 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:18,036 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:18,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742141_1317 (size=14741) 2024-11-07T12:42:18,040 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/e0baccc0f1724da2bf253afb89193399 2024-11-07T12:42:18,049 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/05325fa851b84f4aa9cb074a78b1e97d is 50, key is test_row_0/B:col10/1730983338005/Put/seqid=0 2024-11-07T12:42:18,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-07T12:42:18,052 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:42:18,052 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:42:18,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-07T12:42:18,054 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:42:18,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-07T12:42:18,055 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:42:18,055 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:42:18,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742142_1318 (size=12301) 2024-11-07T12:42:18,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:18,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42034 deadline: 1730983398142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:18,146 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:18,146 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:18,146 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:18,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42026 deadline: 1730983398142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:18,146 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:18,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:18,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:18,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:18,146 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:18,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42070 deadline: 1730983398142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:18,147 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:18,147 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:18,147 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:18,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-07T12:42:18,208 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-07T12:42:18,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:18,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. as already flushing 2024-11-07T12:42:18,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:18,208 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:18,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:18,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:18,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:18,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42034 deadline: 1730983398352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:18,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:18,355 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:18,355 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:18,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42026 deadline: 1730983398352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:18,355 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:18,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:18,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42070 deadline: 1730983398352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:18,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:18,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:18,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:18,356 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:18,356 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:18,356 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:18,364 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-07T12:42:18,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:18,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. as already flushing 2024-11-07T12:42:18,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:18,365 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:18,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:18,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:18,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-07T12:42:18,474 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/05325fa851b84f4aa9cb074a78b1e97d 2024-11-07T12:42:18,500 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/54e8bf43509946a0bdc24aced9b9d8b7 is 50, key is test_row_0/C:col10/1730983338005/Put/seqid=0 2024-11-07T12:42:18,521 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-07T12:42:18,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:18,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. as already flushing 2024-11-07T12:42:18,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:18,525 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:18,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:18,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:18,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742143_1319 (size=12301) 2024-11-07T12:42:18,541 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/54e8bf43509946a0bdc24aced9b9d8b7 2024-11-07T12:42:18,554 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/e0baccc0f1724da2bf253afb89193399 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/e0baccc0f1724da2bf253afb89193399 2024-11-07T12:42:18,563 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/e0baccc0f1724da2bf253afb89193399, entries=200, sequenceid=322, filesize=14.4 K 2024-11-07T12:42:18,564 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/05325fa851b84f4aa9cb074a78b1e97d as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/05325fa851b84f4aa9cb074a78b1e97d 2024-11-07T12:42:18,569 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/05325fa851b84f4aa9cb074a78b1e97d, entries=150, sequenceid=322, filesize=12.0 K 2024-11-07T12:42:18,570 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/54e8bf43509946a0bdc24aced9b9d8b7 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/54e8bf43509946a0bdc24aced9b9d8b7 2024-11-07T12:42:18,576 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/54e8bf43509946a0bdc24aced9b9d8b7, entries=150, sequenceid=322, filesize=12.0 K 2024-11-07T12:42:18,577 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 8dc13886c19c8110f075e43282960321 in 571ms, sequenceid=322, compaction requested=true 2024-11-07T12:42:18,577 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8dc13886c19c8110f075e43282960321: 2024-11-07T12:42:18,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8dc13886c19c8110f075e43282960321:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:42:18,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:42:18,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8dc13886c19c8110f075e43282960321:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:42:18,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-07T12:42:18,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8dc13886c19c8110f075e43282960321:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:42:18,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-07T12:42:18,577 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:42:18,578 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:42:18,579 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40093 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:42:18,579 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 8dc13886c19c8110f075e43282960321/A is initiating minor compaction (all files) 2024-11-07T12:42:18,579 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8dc13886c19c8110f075e43282960321/A in TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:18,579 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/c7ff1c1a9a6143fcba89c5bc815231d6, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/bb7bd17bf41e47b78f8f8ef571c5f6ab, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/e0baccc0f1724da2bf253afb89193399] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp, totalSize=39.2 K 2024-11-07T12:42:18,580 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting c7ff1c1a9a6143fcba89c5bc815231d6, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1730983335935 2024-11-07T12:42:18,580 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting bb7bd17bf41e47b78f8f8ef571c5f6ab, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1730983336588 2024-11-07T12:42:18,581 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting e0baccc0f1724da2bf253afb89193399, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1730983337344 2024-11-07T12:42:18,588 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:42:18,589 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): 8dc13886c19c8110f075e43282960321/C is initiating minor compaction (all files) 2024-11-07T12:42:18,589 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8dc13886c19c8110f075e43282960321/C in TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:18,589 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/5b12a32838434a30bccd3af318159dd0, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/b71dd55b27304789aed52531e7846d9a, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/54e8bf43509946a0bdc24aced9b9d8b7] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp, totalSize=36.8 K 2024-11-07T12:42:18,592 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b12a32838434a30bccd3af318159dd0, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1730983335935 2024-11-07T12:42:18,596 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting b71dd55b27304789aed52531e7846d9a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1730983336588 2024-11-07T12:42:18,597 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54e8bf43509946a0bdc24aced9b9d8b7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1730983337346 2024-11-07T12:42:18,599 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8dc13886c19c8110f075e43282960321#A#compaction#270 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:42:18,599 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/fe649630ed9b4bea9931215aca3fcabf is 50, key is test_row_0/A:col10/1730983338005/Put/seqid=0 2024-11-07T12:42:18,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742144_1320 (size=13153) 2024-11-07T12:42:18,647 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8dc13886c19c8110f075e43282960321#C#compaction#271 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:42:18,647 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/7d327d8bed53415caa6090ef44e67fa2 is 50, key is test_row_0/C:col10/1730983338005/Put/seqid=0 2024-11-07T12:42:18,669 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8dc13886c19c8110f075e43282960321 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-07T12:42:18,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 8dc13886c19c8110f075e43282960321 2024-11-07T12:42:18,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=A 2024-11-07T12:42:18,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:18,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=B 2024-11-07T12:42:18,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:18,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=C 2024-11-07T12:42:18,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:18,681 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/fe649630ed9b4bea9931215aca3fcabf as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/fe649630ed9b4bea9931215aca3fcabf 2024-11-07T12:42:18,682 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-07T12:42:18,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:18,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. as already flushing 2024-11-07T12:42:18,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:18,682 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:18,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:18,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:18,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742145_1321 (size=13153) 2024-11-07T12:42:18,691 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/cfa3d4bc2d7c436eaa27f7c5b7fe98aa is 50, key is test_row_0/A:col10/1730983338667/Put/seqid=0 2024-11-07T12:42:18,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-07T12:42:18,699 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/7d327d8bed53415caa6090ef44e67fa2 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/7d327d8bed53415caa6090ef44e67fa2 2024-11-07T12:42:18,700 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8dc13886c19c8110f075e43282960321/A of 8dc13886c19c8110f075e43282960321 into fe649630ed9b4bea9931215aca3fcabf(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:42:18,700 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8dc13886c19c8110f075e43282960321: 2024-11-07T12:42:18,700 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., storeName=8dc13886c19c8110f075e43282960321/A, priority=13, startTime=1730983338577; duration=0sec 2024-11-07T12:42:18,700 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:42:18,700 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8dc13886c19c8110f075e43282960321:A 2024-11-07T12:42:18,700 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:42:18,706 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:42:18,706 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 8dc13886c19c8110f075e43282960321/B is initiating minor compaction (all files) 2024-11-07T12:42:18,706 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8dc13886c19c8110f075e43282960321/B in TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:18,706 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/613372ea89ec4e6a962c0c708a56f19d, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/519c8d48c87b47ebb01bb2236fb344d5, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/05325fa851b84f4aa9cb074a78b1e97d] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp, totalSize=36.8 K 2024-11-07T12:42:18,707 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 613372ea89ec4e6a962c0c708a56f19d, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1730983335935 2024-11-07T12:42:18,708 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 519c8d48c87b47ebb01bb2236fb344d5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1730983336588 2024-11-07T12:42:18,708 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8dc13886c19c8110f075e43282960321/C of 8dc13886c19c8110f075e43282960321 into 7d327d8bed53415caa6090ef44e67fa2(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:42:18,708 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8dc13886c19c8110f075e43282960321: 2024-11-07T12:42:18,708 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., storeName=8dc13886c19c8110f075e43282960321/C, priority=13, startTime=1730983338577; duration=0sec 2024-11-07T12:42:18,708 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:42:18,708 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 05325fa851b84f4aa9cb074a78b1e97d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1730983337346 2024-11-07T12:42:18,708 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8dc13886c19c8110f075e43282960321:C 2024-11-07T12:42:18,735 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8dc13886c19c8110f075e43282960321#B#compaction#273 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:42:18,736 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/932f336f56794b99bed0d41f0dc1284a is 50, key is test_row_0/B:col10/1730983338005/Put/seqid=0 2024-11-07T12:42:18,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:18,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42034 deadline: 1730983398734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:18,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:18,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:18,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:18,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:18,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42026 deadline: 1730983398734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:18,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:18,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:18,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:18,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:18,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42070 deadline: 1730983398738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:18,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:18,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:18,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:18,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742146_1322 (size=12301) 2024-11-07T12:42:18,782 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/cfa3d4bc2d7c436eaa27f7c5b7fe98aa 2024-11-07T12:42:18,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742147_1323 (size=13153) 2024-11-07T12:42:18,796 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/911434becb874227b5a097efedcc1879 is 50, key is test_row_0/B:col10/1730983338667/Put/seqid=0 2024-11-07T12:42:18,798 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/932f336f56794b99bed0d41f0dc1284a as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/932f336f56794b99bed0d41f0dc1284a 2024-11-07T12:42:18,817 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8dc13886c19c8110f075e43282960321/B of 8dc13886c19c8110f075e43282960321 into 932f336f56794b99bed0d41f0dc1284a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:42:18,817 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8dc13886c19c8110f075e43282960321: 2024-11-07T12:42:18,817 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., storeName=8dc13886c19c8110f075e43282960321/B, priority=13, startTime=1730983338577; duration=0sec 2024-11-07T12:42:18,818 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:42:18,818 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8dc13886c19c8110f075e43282960321:B 2024-11-07T12:42:18,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742148_1324 (size=12301) 2024-11-07T12:42:18,841 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-07T12:42:18,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:18,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. as already flushing 2024-11-07T12:42:18,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:18,841 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:18,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:18,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:18,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:18,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42034 deadline: 1730983398851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:18,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:18,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:18,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:18,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:18,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42026 deadline: 1730983398852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:18,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:18,856 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:18,856 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:18,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42070 deadline: 1730983398852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:18,856 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:18,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:18,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:18,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:18,995 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-07T12:42:18,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:18,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. as already flushing 2024-11-07T12:42:18,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:18,996 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:18,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:19,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:19,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:19,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42026 deadline: 1730983399063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:19,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:19,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42070 deadline: 1730983399063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:19,069 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:19,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42034 deadline: 1730983399064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:19,069 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:19,069 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:19,069 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:19,070 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:19,070 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:19,070 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:19,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:19,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:19,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:19,157 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-07T12:42:19,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:19,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. as already flushing 2024-11-07T12:42:19,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:19,157 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:19,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:19,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:19,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-07T12:42:19,224 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/911434becb874227b5a097efedcc1879 2024-11-07T12:42:19,249 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/f3179830cc594a2596bd5f223cad1b2e is 50, key is test_row_0/C:col10/1730983338667/Put/seqid=0 2024-11-07T12:42:19,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742149_1325 (size=12301) 2024-11-07T12:42:19,268 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/f3179830cc594a2596bd5f223cad1b2e 2024-11-07T12:42:19,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/cfa3d4bc2d7c436eaa27f7c5b7fe98aa as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/cfa3d4bc2d7c436eaa27f7c5b7fe98aa 2024-11-07T12:42:19,284 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/cfa3d4bc2d7c436eaa27f7c5b7fe98aa, entries=150, sequenceid=338, filesize=12.0 K 2024-11-07T12:42:19,284 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/911434becb874227b5a097efedcc1879 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/911434becb874227b5a097efedcc1879 2024-11-07T12:42:19,290 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/911434becb874227b5a097efedcc1879, entries=150, sequenceid=338, filesize=12.0 K 2024-11-07T12:42:19,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/f3179830cc594a2596bd5f223cad1b2e as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/f3179830cc594a2596bd5f223cad1b2e 2024-11-07T12:42:19,295 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/f3179830cc594a2596bd5f223cad1b2e, entries=150, sequenceid=338, filesize=12.0 K 2024-11-07T12:42:19,296 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 8dc13886c19c8110f075e43282960321 in 628ms, sequenceid=338, compaction requested=false 2024-11-07T12:42:19,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8dc13886c19c8110f075e43282960321: 2024-11-07T12:42:19,312 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-07T12:42:19,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:19,313 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 8dc13886c19c8110f075e43282960321 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-07T12:42:19,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=A 2024-11-07T12:42:19,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:19,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=B 2024-11-07T12:42:19,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:19,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=C 2024-11-07T12:42:19,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:19,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/83d995e5e424409ba9842c59786264d7 is 50, key is test_row_0/A:col10/1730983338728/Put/seqid=0 2024-11-07T12:42:19,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742150_1326 (size=12301) 2024-11-07T12:42:19,327 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/83d995e5e424409ba9842c59786264d7 2024-11-07T12:42:19,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/cf774d9d1b3a47dd99d9cdf187c714c6 is 50, key is test_row_0/B:col10/1730983338728/Put/seqid=0 2024-11-07T12:42:19,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742151_1327 (size=12301) 2024-11-07T12:42:19,375 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/cf774d9d1b3a47dd99d9cdf187c714c6 2024-11-07T12:42:19,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 8dc13886c19c8110f075e43282960321 2024-11-07T12:42:19,383 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. as already flushing 2024-11-07T12:42:19,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/b166f92db6924e0c8737c4ad6df85437 is 50, key is test_row_0/C:col10/1730983338728/Put/seqid=0 2024-11-07T12:42:19,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742152_1328 (size=12301) 2024-11-07T12:42:19,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:19,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42026 deadline: 1730983399408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:19,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:19,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42070 deadline: 1730983399409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:19,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:19,415 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/b166f92db6924e0c8737c4ad6df85437 2024-11-07T12:42:19,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:19,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:19,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:19,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:19,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:19,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/83d995e5e424409ba9842c59786264d7 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/83d995e5e424409ba9842c59786264d7 2024-11-07T12:42:19,425 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/83d995e5e424409ba9842c59786264d7, entries=150, sequenceid=357, filesize=12.0 K 2024-11-07T12:42:19,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/cf774d9d1b3a47dd99d9cdf187c714c6 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/cf774d9d1b3a47dd99d9cdf187c714c6 2024-11-07T12:42:19,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:19,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42034 deadline: 1730983399429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:19,432 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/cf774d9d1b3a47dd99d9cdf187c714c6, entries=150, sequenceid=357, filesize=12.0 K 2024-11-07T12:42:19,433 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:19,433 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:19,433 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:19,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/b166f92db6924e0c8737c4ad6df85437 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/b166f92db6924e0c8737c4ad6df85437 2024-11-07T12:42:19,438 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/b166f92db6924e0c8737c4ad6df85437, entries=150, sequenceid=357, filesize=12.0 K 2024-11-07T12:42:19,439 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 8dc13886c19c8110f075e43282960321 in 126ms, sequenceid=357, compaction requested=true 2024-11-07T12:42:19,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 8dc13886c19c8110f075e43282960321: 2024-11-07T12:42:19,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:19,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-07T12:42:19,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-07T12:42:19,443 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-07T12:42:19,443 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3860 sec 2024-11-07T12:42:19,445 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.3910 sec 2024-11-07T12:42:19,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 8dc13886c19c8110f075e43282960321 2024-11-07T12:42:19,526 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8dc13886c19c8110f075e43282960321 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-07T12:42:19,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=A 2024-11-07T12:42:19,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:19,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=B 2024-11-07T12:42:19,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:19,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=C 2024-11-07T12:42:19,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:19,544 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/932f933799264cdab336d4487c6b53b3 is 50, key is test_row_0/A:col10/1730983339524/Put/seqid=0 2024-11-07T12:42:19,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742153_1329 (size=12301) 2024-11-07T12:42:19,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:19,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42054 deadline: 1730983399559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:19,567 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/932f933799264cdab336d4487c6b53b3 2024-11-07T12:42:19,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:19,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:19,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:19,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:19,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41976 deadline: 1730983399560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:19,570 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:19,570 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:19,570 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:19,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:19,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42034 deadline: 1730983399563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:19,572 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:19,572 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:19,572 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:19,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:19,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42026 deadline: 1730983399568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:19,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:19,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42070 deadline: 1730983399570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:19,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:19,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:19,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:19,578 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:19,578 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:19,578 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:19,601 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/bc8f460ec43a4c06aa37fe8f7994d1c4 is 50, key is test_row_0/B:col10/1730983339524/Put/seqid=0 2024-11-07T12:42:19,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742154_1330 (size=12301) 2024-11-07T12:42:19,624 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/bc8f460ec43a4c06aa37fe8f7994d1c4 2024-11-07T12:42:19,651 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/6ede7d2abb66452e84ddbc6332ae69e1 is 50, key is test_row_0/C:col10/1730983339524/Put/seqid=0 2024-11-07T12:42:19,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:19,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41976 deadline: 1730983399672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:19,678 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:19,678 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:19,678 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:19,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:19,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42054 deadline: 1730983399675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:19,680 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:19,680 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:19,680 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:19,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:19,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42034 deadline: 1730983399682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:19,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:19,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42070 deadline: 1730983399682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:19,691 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:19,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:19,691 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:19,691 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:19,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42026 deadline: 1730983399682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:19,691 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:19,691 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:19,691 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:19,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:19,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:19,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:19,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742155_1331 (size=12301) 2024-11-07T12:42:19,717 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/6ede7d2abb66452e84ddbc6332ae69e1 2024-11-07T12:42:19,784 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/932f933799264cdab336d4487c6b53b3 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/932f933799264cdab336d4487c6b53b3 2024-11-07T12:42:19,837 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/932f933799264cdab336d4487c6b53b3, entries=150, sequenceid=374, filesize=12.0 K 2024-11-07T12:42:19,848 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/bc8f460ec43a4c06aa37fe8f7994d1c4 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/bc8f460ec43a4c06aa37fe8f7994d1c4 2024-11-07T12:42:19,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:19,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41976 deadline: 1730983399881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:19,889 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:19,889 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:19,889 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:19,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:19,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42054 deadline: 1730983399888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:19,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:19,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:19,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:19,907 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/bc8f460ec43a4c06aa37fe8f7994d1c4, entries=150, sequenceid=374, filesize=12.0 K 2024-11-07T12:42:19,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:19,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42034 deadline: 1730983399905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:19,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:19,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42070 deadline: 1730983399906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:19,910 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:19,910 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:19,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:19,910 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:19,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42026 deadline: 1730983399906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:19,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:19,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:19,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:19,911 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:19,911 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:19,911 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:19,924 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/6ede7d2abb66452e84ddbc6332ae69e1 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/6ede7d2abb66452e84ddbc6332ae69e1 2024-11-07T12:42:19,975 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/6ede7d2abb66452e84ddbc6332ae69e1, entries=150, sequenceid=374, filesize=12.0 K 2024-11-07T12:42:19,984 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for 8dc13886c19c8110f075e43282960321 in 459ms, sequenceid=374, compaction requested=true 2024-11-07T12:42:19,985 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8dc13886c19c8110f075e43282960321: 2024-11-07T12:42:19,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8dc13886c19c8110f075e43282960321:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:42:19,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:42:19,985 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:42:19,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8dc13886c19c8110f075e43282960321:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:42:19,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:42:19,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8dc13886c19c8110f075e43282960321:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:42:19,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-07T12:42:19,988 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:42:20,024 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50056 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:42:20,025 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 8dc13886c19c8110f075e43282960321/A is initiating minor compaction (all files) 2024-11-07T12:42:20,025 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8dc13886c19c8110f075e43282960321/A in TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:20,025 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/fe649630ed9b4bea9931215aca3fcabf, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/cfa3d4bc2d7c436eaa27f7c5b7fe98aa, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/83d995e5e424409ba9842c59786264d7, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/932f933799264cdab336d4487c6b53b3] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp, totalSize=48.9 K 2024-11-07T12:42:20,027 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50056 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:42:20,027 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): 8dc13886c19c8110f075e43282960321/B is initiating minor compaction (all files) 2024-11-07T12:42:20,027 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8dc13886c19c8110f075e43282960321/B in TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:20,027 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/932f336f56794b99bed0d41f0dc1284a, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/911434becb874227b5a097efedcc1879, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/cf774d9d1b3a47dd99d9cdf187c714c6, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/bc8f460ec43a4c06aa37fe8f7994d1c4] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp, totalSize=48.9 K 2024-11-07T12:42:20,032 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting fe649630ed9b4bea9931215aca3fcabf, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1730983337346 2024-11-07T12:42:20,036 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 932f336f56794b99bed0d41f0dc1284a, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1730983337346 2024-11-07T12:42:20,044 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting cfa3d4bc2d7c436eaa27f7c5b7fe98aa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1730983338029 2024-11-07T12:42:20,046 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 911434becb874227b5a097efedcc1879, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1730983338029 2024-11-07T12:42:20,048 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 83d995e5e424409ba9842c59786264d7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1730983338715 2024-11-07T12:42:20,051 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf774d9d1b3a47dd99d9cdf187c714c6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1730983338715 2024-11-07T12:42:20,053 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 932f933799264cdab336d4487c6b53b3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1730983339406 2024-11-07T12:42:20,057 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc8f460ec43a4c06aa37fe8f7994d1c4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1730983339406 2024-11-07T12:42:20,156 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8dc13886c19c8110f075e43282960321#A#compaction#282 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:42:20,157 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/a0a1958157724b779615a421d569a2cf is 50, key is test_row_0/A:col10/1730983339524/Put/seqid=0 2024-11-07T12:42:20,161 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8dc13886c19c8110f075e43282960321#B#compaction#283 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:42:20,162 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/4a415639b238442983553587eeec51ce is 50, key is test_row_0/B:col10/1730983339524/Put/seqid=0 2024-11-07T12:42:20,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742156_1332 (size=13289) 2024-11-07T12:42:20,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742157_1333 (size=13289) 2024-11-07T12:42:20,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 8dc13886c19c8110f075e43282960321 2024-11-07T12:42:20,199 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8dc13886c19c8110f075e43282960321 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-07T12:42:20,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=A 2024-11-07T12:42:20,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:20,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=B 2024-11-07T12:42:20,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:20,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=C 2024-11-07T12:42:20,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:20,202 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/a0a1958157724b779615a421d569a2cf as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/a0a1958157724b779615a421d569a2cf 2024-11-07T12:42:20,205 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/9395bc02a5734172815e7119d3ece447 is 50, key is test_row_0/A:col10/1730983339558/Put/seqid=0 2024-11-07T12:42:20,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-07T12:42:20,213 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8dc13886c19c8110f075e43282960321/A of 8dc13886c19c8110f075e43282960321 into a0a1958157724b779615a421d569a2cf(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:42:20,213 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8dc13886c19c8110f075e43282960321: 2024-11-07T12:42:20,213 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., storeName=8dc13886c19c8110f075e43282960321/A, priority=12, startTime=1730983339985; duration=0sec 2024-11-07T12:42:20,214 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:42:20,214 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8dc13886c19c8110f075e43282960321:A 2024-11-07T12:42:20,214 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:42:20,216 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:42:20,216 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50056 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:42:20,216 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 8dc13886c19c8110f075e43282960321/C is initiating minor compaction (all files) 2024-11-07T12:42:20,216 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8dc13886c19c8110f075e43282960321/C in TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:20,216 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/7d327d8bed53415caa6090ef44e67fa2, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/f3179830cc594a2596bd5f223cad1b2e, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/b166f92db6924e0c8737c4ad6df85437, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/6ede7d2abb66452e84ddbc6332ae69e1] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp, totalSize=48.9 K 2024-11-07T12:42:20,216 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:42:20,217 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d327d8bed53415caa6090ef44e67fa2, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1730983337346 2024-11-07T12:42:20,217 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting f3179830cc594a2596bd5f223cad1b2e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1730983338029 2024-11-07T12:42:20,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-07T12:42:20,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-07T12:42:20,218 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:42:20,218 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting b166f92db6924e0c8737c4ad6df85437, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1730983338715 2024-11-07T12:42:20,219 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ede7d2abb66452e84ddbc6332ae69e1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1730983339406 2024-11-07T12:42:20,219 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:42:20,219 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:42:20,236 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8dc13886c19c8110f075e43282960321#C#compaction#285 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:42:20,236 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/2527a28eabfe429d9f9a7c6ef917bcb2 is 50, key is test_row_0/C:col10/1730983339524/Put/seqid=0 2024-11-07T12:42:20,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742158_1334 (size=14741) 2024-11-07T12:42:20,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:20,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42026 deadline: 1730983400234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:20,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742159_1335 (size=13289) 2024-11-07T12:42:20,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:20,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42034 deadline: 1730983400234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:20,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:20,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:20,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:20,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42054 deadline: 1730983400235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:20,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:20,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:20,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42070 deadline: 1730983400237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:20,243 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:20,243 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:20,243 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:20,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:20,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:20,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:20,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:20,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41976 deadline: 1730983400246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:20,248 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:20,248 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:20,249 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:20,251 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:20,251 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:20,251 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:20,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-07T12:42:20,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:20,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42054 deadline: 1730983400352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:20,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:20,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42034 deadline: 1730983400352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:20,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:20,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42026 deadline: 1730983400353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:20,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:20,357 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:20,357 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:20,357 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:20,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42070 deadline: 1730983400353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:20,357 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:20,357 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:20,357 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:20,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:20,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:20,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:20,357 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:20,357 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:20,357 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:20,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:20,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41976 deadline: 1730983400362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:20,364 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:20,364 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:20,364 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:20,372 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-07T12:42:20,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:20,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. as already flushing 2024-11-07T12:42:20,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:20,372 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:20,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:20,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:20,526 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-07T12:42:20,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:20,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. as already flushing 2024-11-07T12:42:20,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:20,527 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:20,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:20,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:20,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-07T12:42:20,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:20,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42034 deadline: 1730983400562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:20,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:20,567 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:20,567 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:20,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42070 deadline: 1730983400562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:20,567 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:20,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:20,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:20,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:20,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42026 deadline: 1730983400562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:20,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:20,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:20,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42054 deadline: 1730983400562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:20,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:20,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:20,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:20,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:20,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:20,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:20,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:20,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41976 deadline: 1730983400571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:20,576 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:20,576 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:20,576 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:20,602 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/4a415639b238442983553587eeec51ce as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/4a415639b238442983553587eeec51ce 2024-11-07T12:42:20,607 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8dc13886c19c8110f075e43282960321/B of 8dc13886c19c8110f075e43282960321 into 4a415639b238442983553587eeec51ce(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:42:20,607 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8dc13886c19c8110f075e43282960321: 2024-11-07T12:42:20,607 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., storeName=8dc13886c19c8110f075e43282960321/B, priority=12, startTime=1730983339985; duration=0sec 2024-11-07T12:42:20,607 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:42:20,607 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8dc13886c19c8110f075e43282960321:B 2024-11-07T12:42:20,639 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/9395bc02a5734172815e7119d3ece447 2024-11-07T12:42:20,647 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/7a97b9dbc5414a6ca69cfbb0a758f6f2 is 50, key is test_row_0/B:col10/1730983339558/Put/seqid=0 2024-11-07T12:42:20,654 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/2527a28eabfe429d9f9a7c6ef917bcb2 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/2527a28eabfe429d9f9a7c6ef917bcb2 2024-11-07T12:42:20,664 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8dc13886c19c8110f075e43282960321/C of 8dc13886c19c8110f075e43282960321 into 2527a28eabfe429d9f9a7c6ef917bcb2(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:42:20,664 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8dc13886c19c8110f075e43282960321: 2024-11-07T12:42:20,664 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., storeName=8dc13886c19c8110f075e43282960321/C, priority=12, startTime=1730983339985; duration=0sec 2024-11-07T12:42:20,664 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:42:20,664 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8dc13886c19c8110f075e43282960321:C 2024-11-07T12:42:20,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742160_1336 (size=12301) 2024-11-07T12:42:20,676 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/7a97b9dbc5414a6ca69cfbb0a758f6f2 2024-11-07T12:42:20,689 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/90bbd66496184bc2be09eee5a79a5efe is 50, key is test_row_0/C:col10/1730983339558/Put/seqid=0 2024-11-07T12:42:20,693 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-07T12:42:20,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:20,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. as already flushing 2024-11-07T12:42:20,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:20,693 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:20,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:20,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:20,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742161_1337 (size=12301) 2024-11-07T12:42:20,700 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/90bbd66496184bc2be09eee5a79a5efe 2024-11-07T12:42:20,707 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/9395bc02a5734172815e7119d3ece447 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/9395bc02a5734172815e7119d3ece447 2024-11-07T12:42:20,712 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/9395bc02a5734172815e7119d3ece447, entries=200, sequenceid=391, filesize=14.4 K 2024-11-07T12:42:20,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/7a97b9dbc5414a6ca69cfbb0a758f6f2 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/7a97b9dbc5414a6ca69cfbb0a758f6f2 2024-11-07T12:42:20,722 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/7a97b9dbc5414a6ca69cfbb0a758f6f2, entries=150, sequenceid=391, filesize=12.0 K 2024-11-07T12:42:20,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/90bbd66496184bc2be09eee5a79a5efe as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/90bbd66496184bc2be09eee5a79a5efe 2024-11-07T12:42:20,727 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/90bbd66496184bc2be09eee5a79a5efe, entries=150, sequenceid=391, filesize=12.0 K 2024-11-07T12:42:20,728 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 8dc13886c19c8110f075e43282960321 in 529ms, sequenceid=391, compaction requested=false 2024-11-07T12:42:20,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8dc13886c19c8110f075e43282960321: 2024-11-07T12:42:20,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-07T12:42:20,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-07T12:42:20,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:20,853 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 8dc13886c19c8110f075e43282960321 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-07T12:42:20,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=A 2024-11-07T12:42:20,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:20,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=B 2024-11-07T12:42:20,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:20,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=C 2024-11-07T12:42:20,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:20,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/43372c6fdbe2423b81daac3750155cf2 is 50, key is test_row_0/A:col10/1730983340232/Put/seqid=0 2024-11-07T12:42:20,876 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. as already flushing 2024-11-07T12:42:20,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 8dc13886c19c8110f075e43282960321 2024-11-07T12:42:20,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742162_1338 (size=12301) 2024-11-07T12:42:20,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:20,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42034 deadline: 1730983400892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:20,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:20,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42070 deadline: 1730983400897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:20,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:20,901 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:20,901 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:20,901 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:20,901 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:20,901 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:20,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:20,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42054 deadline: 1730983400898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:20,903 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:20,903 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:20,903 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:20,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:20,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42026 deadline: 1730983400900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:20,905 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:20,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:20,905 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:20,905 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:20,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41976 deadline: 1730983400901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:20,906 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:20,906 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:20,906 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:21,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:21,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42034 deadline: 1730983401011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:21,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:21,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:21,016 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:21,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41976 deadline: 1730983401012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:21,016 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:21,016 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:21,016 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:21,016 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:21,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:21,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42070 deadline: 1730983401012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:21,016 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:21,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:21,016 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:21,016 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:21,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42054 deadline: 1730983401012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:21,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:21,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:21,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:21,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:21,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42026 deadline: 1730983401012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:21,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:21,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:21,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:21,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:21,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42026 deadline: 1730983401222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:21,226 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:21,226 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:21,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:21,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:21,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:21,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42034 deadline: 1730983401222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:21,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42054 deadline: 1730983401223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:21,227 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:21,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:21,227 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:21,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:21,227 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:21,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:21,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:21,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41976 deadline: 1730983401228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:21,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:21,229 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:21,229 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:21,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42070 deadline: 1730983401229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:21,229 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:21,230 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:21,230 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:21,230 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:21,295 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/43372c6fdbe2423b81daac3750155cf2 2024-11-07T12:42:21,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/0263d7c342ce48158ac803795745cc13 is 50, key is test_row_0/B:col10/1730983340232/Put/seqid=0 2024-11-07T12:42:21,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-07T12:42:21,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742163_1339 (size=12301) 2024-11-07T12:42:21,377 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/0263d7c342ce48158ac803795745cc13 2024-11-07T12:42:21,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/d7e039431fee4462bb3c1a01ca53a0ff is 50, key is test_row_0/C:col10/1730983340232/Put/seqid=0 2024-11-07T12:42:21,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742164_1340 (size=12301) 2024-11-07T12:42:21,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:21,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42034 deadline: 1730983401532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:21,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:21,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:21,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:21,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:21,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42026 deadline: 1730983401532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:21,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:21,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:21,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:21,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:21,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41976 deadline: 1730983401532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:21,536 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:21,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:21,536 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:21,536 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:21,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42054 deadline: 1730983401533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:21,536 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:21,536 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:21,536 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:21,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:21,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42070 deadline: 1730983401543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:21,548 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:21,548 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:21,548 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:21,822 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/d7e039431fee4462bb3c1a01ca53a0ff 2024-11-07T12:42:21,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/43372c6fdbe2423b81daac3750155cf2 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/43372c6fdbe2423b81daac3750155cf2 2024-11-07T12:42:21,835 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/43372c6fdbe2423b81daac3750155cf2, entries=150, sequenceid=409, filesize=12.0 K 2024-11-07T12:42:21,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/0263d7c342ce48158ac803795745cc13 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/0263d7c342ce48158ac803795745cc13 2024-11-07T12:42:21,840 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/0263d7c342ce48158ac803795745cc13, entries=150, sequenceid=409, filesize=12.0 K 2024-11-07T12:42:21,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/d7e039431fee4462bb3c1a01ca53a0ff as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/d7e039431fee4462bb3c1a01ca53a0ff 2024-11-07T12:42:21,850 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/d7e039431fee4462bb3c1a01ca53a0ff, entries=150, sequenceid=409, filesize=12.0 K 2024-11-07T12:42:21,851 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 8dc13886c19c8110f075e43282960321 in 998ms, sequenceid=409, compaction requested=true 2024-11-07T12:42:21,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 8dc13886c19c8110f075e43282960321: 2024-11-07T12:42:21,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:21,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-07T12:42:21,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-07T12:42:21,856 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-07T12:42:21,856 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6340 sec 2024-11-07T12:42:21,859 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.6410 sec 2024-11-07T12:42:22,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 8dc13886c19c8110f075e43282960321 2024-11-07T12:42:22,044 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8dc13886c19c8110f075e43282960321 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-07T12:42:22,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=A 2024-11-07T12:42:22,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:22,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=B 2024-11-07T12:42:22,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:22,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=C 2024-11-07T12:42:22,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:22,052 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/2e392408f4ca462ba870f1c15c92f4d0 is 50, key is test_row_0/A:col10/1730983342043/Put/seqid=0 2024-11-07T12:42:22,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742165_1341 (size=14741) 2024-11-07T12:42:22,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:22,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42034 deadline: 1730983402062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:22,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:22,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:22,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42026 deadline: 1730983402063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:22,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41976 deadline: 1730983402064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:22,071 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:22,071 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:22,071 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:22,072 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:22,072 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:22,072 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:22,073 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:22,073 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:22,073 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:22,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:22,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42054 deadline: 1730983402072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:22,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:22,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42070 deadline: 1730983402073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:22,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:22,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:22,077 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:22,077 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:22,077 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:22,077 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:22,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:22,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41976 deadline: 1730983402183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:22,187 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:22,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:22,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:22,187 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:22,188 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:22,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42070 deadline: 1730983402183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:22,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42034 deadline: 1730983402183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:22,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:22,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:22,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:22,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:22,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:22,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:22,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:22,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42054 deadline: 1730983402183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:22,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:22,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:22,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:22,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:22,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42026 deadline: 1730983402186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:22,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:22,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:22,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:22,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-07T12:42:22,373 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:42:22,376 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:42:22,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-07T12:42:22,385 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:42:22,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-07T12:42:22,386 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:42:22,386 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:42:22,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:22,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41976 deadline: 1730983402391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:22,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:22,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42026 deadline: 1730983402392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:22,397 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:22,397 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:22,397 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:22,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:22,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:22,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42070 deadline: 1730983402392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:22,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:22,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:22,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:22,398 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:22,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42034 deadline: 1730983402392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:22,398 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:22,398 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:22,398 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:22,398 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:22,398 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:22,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:22,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42054 deadline: 1730983402392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:22,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:22,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:22,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:22,476 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/2e392408f4ca462ba870f1c15c92f4d0 2024-11-07T12:42:22,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-07T12:42:22,523 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/47d126917a7045d8843c3eb1f4b5071c is 50, key is test_row_0/B:col10/1730983342043/Put/seqid=0 2024-11-07T12:42:22,545 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-07T12:42:22,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:22,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. as already flushing 2024-11-07T12:42:22,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:22,546 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:22,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:22,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:22,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742166_1342 (size=12301) 2024-11-07T12:42:22,570 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/47d126917a7045d8843c3eb1f4b5071c 2024-11-07T12:42:22,619 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/25a8f16d8b6a4623be5fd581539121db is 50, key is test_row_0/C:col10/1730983342043/Put/seqid=0 2024-11-07T12:42:22,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742167_1343 (size=12301) 2024-11-07T12:42:22,704 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-07T12:42:22,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:22,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. as already flushing 2024-11-07T12:42:22,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:22,704 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:22,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:22,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:22,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:22,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41976 deadline: 1730983402703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:22,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:22,708 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:22,708 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:22,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42026 deadline: 1730983402703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:22,708 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:22,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:22,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42070 deadline: 1730983402705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:22,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:22,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:22,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:22,712 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:22,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-07T12:42:22,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:22,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:22,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:22,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42034 deadline: 1730983402711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:22,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:22,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:22,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:22,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:22,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42054 deadline: 1730983402712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:22,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:22,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:22,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:22,857 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-07T12:42:22,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:22,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. as already flushing 2024-11-07T12:42:22,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:22,857 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:22,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:22,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:23,015 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-07T12:42:23,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:23,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. as already flushing 2024-11-07T12:42:23,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:23,017 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:23,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:23,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:23,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-07T12:42:23,065 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/25a8f16d8b6a4623be5fd581539121db 2024-11-07T12:42:23,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/2e392408f4ca462ba870f1c15c92f4d0 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/2e392408f4ca462ba870f1c15c92f4d0 2024-11-07T12:42:23,084 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/2e392408f4ca462ba870f1c15c92f4d0, entries=200, sequenceid=428, filesize=14.4 K 2024-11-07T12:42:23,085 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/47d126917a7045d8843c3eb1f4b5071c as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/47d126917a7045d8843c3eb1f4b5071c 2024-11-07T12:42:23,089 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/47d126917a7045d8843c3eb1f4b5071c, entries=150, sequenceid=428, filesize=12.0 K 2024-11-07T12:42:23,090 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/25a8f16d8b6a4623be5fd581539121db as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/25a8f16d8b6a4623be5fd581539121db 2024-11-07T12:42:23,096 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/25a8f16d8b6a4623be5fd581539121db, entries=150, sequenceid=428, filesize=12.0 K 2024-11-07T12:42:23,101 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 8dc13886c19c8110f075e43282960321 in 1057ms, sequenceid=428, compaction requested=true 2024-11-07T12:42:23,101 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8dc13886c19c8110f075e43282960321: 2024-11-07T12:42:23,101 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:42:23,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8dc13886c19c8110f075e43282960321:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:42:23,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:42:23,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8dc13886c19c8110f075e43282960321:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:42:23,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:42:23,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8dc13886c19c8110f075e43282960321:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:42:23,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:42:23,102 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:42:23,104 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50192 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:42:23,104 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55072 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:42:23,104 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): 8dc13886c19c8110f075e43282960321/A is initiating minor compaction (all files) 2024-11-07T12:42:23,104 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 8dc13886c19c8110f075e43282960321/B is initiating minor compaction (all files) 2024-11-07T12:42:23,105 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8dc13886c19c8110f075e43282960321/A in TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:23,105 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/a0a1958157724b779615a421d569a2cf, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/9395bc02a5734172815e7119d3ece447, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/43372c6fdbe2423b81daac3750155cf2, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/2e392408f4ca462ba870f1c15c92f4d0] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp, totalSize=53.8 K 2024-11-07T12:42:23,105 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8dc13886c19c8110f075e43282960321/B in TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:23,105 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/4a415639b238442983553587eeec51ce, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/7a97b9dbc5414a6ca69cfbb0a758f6f2, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/0263d7c342ce48158ac803795745cc13, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/47d126917a7045d8843c3eb1f4b5071c] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp, totalSize=49.0 K 2024-11-07T12:42:23,106 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a415639b238442983553587eeec51ce, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1730983339406 2024-11-07T12:42:23,106 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0a1958157724b779615a421d569a2cf, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1730983339406 2024-11-07T12:42:23,107 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a97b9dbc5414a6ca69cfbb0a758f6f2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1730983339558 2024-11-07T12:42:23,107 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9395bc02a5734172815e7119d3ece447, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1730983339558 2024-11-07T12:42:23,107 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 0263d7c342ce48158ac803795745cc13, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1730983340231 2024-11-07T12:42:23,107 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43372c6fdbe2423b81daac3750155cf2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1730983340231 2024-11-07T12:42:23,107 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e392408f4ca462ba870f1c15c92f4d0, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1730983340890 2024-11-07T12:42:23,108 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 47d126917a7045d8843c3eb1f4b5071c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1730983340890 2024-11-07T12:42:23,131 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8dc13886c19c8110f075e43282960321#A#compaction#294 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:42:23,132 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/0949ef2b947e4f63900de72628f3a73f is 50, key is test_row_0/A:col10/1730983342043/Put/seqid=0 2024-11-07T12:42:23,147 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8dc13886c19c8110f075e43282960321#B#compaction#295 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:42:23,148 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/6f89040571cb4bda9aab6eb7e8d461d3 is 50, key is test_row_0/B:col10/1730983342043/Put/seqid=0 2024-11-07T12:42:23,170 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-07T12:42:23,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:23,170 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing 8dc13886c19c8110f075e43282960321 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-07T12:42:23,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=A 2024-11-07T12:42:23,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:23,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=B 2024-11-07T12:42:23,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:23,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=C 2024-11-07T12:42:23,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:23,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742168_1344 (size=13425) 2024-11-07T12:42:23,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/25e5e7eacdf04a62a55257ec1ab749ac is 50, key is test_row_0/A:col10/1730983342069/Put/seqid=0 2024-11-07T12:42:23,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742169_1345 (size=13425) 2024-11-07T12:42:23,214 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/6f89040571cb4bda9aab6eb7e8d461d3 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/6f89040571cb4bda9aab6eb7e8d461d3 2024-11-07T12:42:23,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 8dc13886c19c8110f075e43282960321 2024-11-07T12:42:23,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. as already flushing 2024-11-07T12:42:23,224 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8dc13886c19c8110f075e43282960321/B of 8dc13886c19c8110f075e43282960321 into 6f89040571cb4bda9aab6eb7e8d461d3(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:42:23,224 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8dc13886c19c8110f075e43282960321: 2024-11-07T12:42:23,224 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., storeName=8dc13886c19c8110f075e43282960321/B, priority=12, startTime=1730983343102; duration=0sec 2024-11-07T12:42:23,224 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:42:23,224 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8dc13886c19c8110f075e43282960321:B 2024-11-07T12:42:23,224 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:42:23,226 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50192 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:42:23,226 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 8dc13886c19c8110f075e43282960321/C is initiating minor compaction (all files) 2024-11-07T12:42:23,226 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8dc13886c19c8110f075e43282960321/C in TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:23,226 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/2527a28eabfe429d9f9a7c6ef917bcb2, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/90bbd66496184bc2be09eee5a79a5efe, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/d7e039431fee4462bb3c1a01ca53a0ff, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/25a8f16d8b6a4623be5fd581539121db] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp, totalSize=49.0 K 2024-11-07T12:42:23,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742170_1346 (size=12301) 2024-11-07T12:42:23,227 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 2527a28eabfe429d9f9a7c6ef917bcb2, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1730983339406 2024-11-07T12:42:23,228 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 90bbd66496184bc2be09eee5a79a5efe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1730983339558 2024-11-07T12:42:23,228 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting d7e039431fee4462bb3c1a01ca53a0ff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1730983340231 2024-11-07T12:42:23,228 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 25a8f16d8b6a4623be5fd581539121db, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1730983340890 2024-11-07T12:42:23,232 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/25e5e7eacdf04a62a55257ec1ab749ac 2024-11-07T12:42:23,242 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8dc13886c19c8110f075e43282960321#C#compaction#298 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:42:23,243 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/29484e2d8a874546958567a53a29b7ec is 50, key is test_row_0/C:col10/1730983342043/Put/seqid=0 2024-11-07T12:42:23,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/c4e1484e0ef34006bcb6c892ea7ebd0b is 50, key is test_row_0/B:col10/1730983342069/Put/seqid=0 2024-11-07T12:42:23,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742171_1347 (size=12301) 2024-11-07T12:42:23,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742172_1348 (size=13425) 2024-11-07T12:42:23,267 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/29484e2d8a874546958567a53a29b7ec as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/29484e2d8a874546958567a53a29b7ec 2024-11-07T12:42:23,279 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8dc13886c19c8110f075e43282960321/C of 8dc13886c19c8110f075e43282960321 into 29484e2d8a874546958567a53a29b7ec(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:42:23,279 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8dc13886c19c8110f075e43282960321: 2024-11-07T12:42:23,279 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., storeName=8dc13886c19c8110f075e43282960321/C, priority=12, startTime=1730983343102; duration=0sec 2024-11-07T12:42:23,280 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:42:23,280 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8dc13886c19c8110f075e43282960321:C 2024-11-07T12:42:23,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:23,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:23,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41976 deadline: 1730983403289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:23,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42034 deadline: 1730983403291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:23,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:23,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42070 deadline: 1730983403293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:23,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:23,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42026 deadline: 1730983403294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:23,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:23,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:23,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:23,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:23,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42054 deadline: 1730983403295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:23,301 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:23,301 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:23,301 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:23,301 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:23,301 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:23,301 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:23,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:23,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:23,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:23,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:23,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:23,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:23,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:23,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42070 deadline: 1730983403411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:23,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:23,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:23,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41976 deadline: 1730983403412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:23,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:23,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:23,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:23,415 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:23,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42054 deadline: 1730983403412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:23,415 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:23,415 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:23,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:23,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:23,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:23,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42026 deadline: 1730983403412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:23,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:23,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:23,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:23,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:23,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:23,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42034 deadline: 1730983403412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:23,416 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:23,416 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:23,416 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:23,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-07T12:42:23,588 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/0949ef2b947e4f63900de72628f3a73f as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/0949ef2b947e4f63900de72628f3a73f 2024-11-07T12:42:23,593 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8dc13886c19c8110f075e43282960321/A of 8dc13886c19c8110f075e43282960321 into 0949ef2b947e4f63900de72628f3a73f(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:42:23,593 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8dc13886c19c8110f075e43282960321: 2024-11-07T12:42:23,593 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., storeName=8dc13886c19c8110f075e43282960321/A, priority=12, startTime=1730983343101; duration=0sec 2024-11-07T12:42:23,593 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:42:23,593 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8dc13886c19c8110f075e43282960321:A 2024-11-07T12:42:23,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:23,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42070 deadline: 1730983403622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:23,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:23,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:23,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:23,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41976 deadline: 1730983403622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:23,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:23,626 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:23,626 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:23,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:23,626 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:23,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42054 deadline: 1730983403622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:23,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:23,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:23,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:23,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:23,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42026 deadline: 1730983403623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:23,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:23,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42034 deadline: 1730983403623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:23,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:23,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:23,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:23,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:23,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:23,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:23,662 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/c4e1484e0ef34006bcb6c892ea7ebd0b 2024-11-07T12:42:23,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/04470e32f61a4f56b54a9c7e5b655ac8 is 50, key is test_row_0/C:col10/1730983342069/Put/seqid=0 2024-11-07T12:42:23,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742173_1349 (size=12301) 2024-11-07T12:42:23,683 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/04470e32f61a4f56b54a9c7e5b655ac8 2024-11-07T12:42:23,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/25e5e7eacdf04a62a55257ec1ab749ac as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/25e5e7eacdf04a62a55257ec1ab749ac 2024-11-07T12:42:23,692 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/25e5e7eacdf04a62a55257ec1ab749ac, entries=150, sequenceid=442, filesize=12.0 K 2024-11-07T12:42:23,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/c4e1484e0ef34006bcb6c892ea7ebd0b as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/c4e1484e0ef34006bcb6c892ea7ebd0b 2024-11-07T12:42:23,697 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/c4e1484e0ef34006bcb6c892ea7ebd0b, entries=150, sequenceid=442, filesize=12.0 K 2024-11-07T12:42:23,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/04470e32f61a4f56b54a9c7e5b655ac8 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/04470e32f61a4f56b54a9c7e5b655ac8 2024-11-07T12:42:23,703 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/04470e32f61a4f56b54a9c7e5b655ac8, entries=150, sequenceid=442, filesize=12.0 K 2024-11-07T12:42:23,704 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=114.05 KB/116790 for 8dc13886c19c8110f075e43282960321 in 534ms, sequenceid=442, compaction requested=false 2024-11-07T12:42:23,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 8dc13886c19c8110f075e43282960321: 2024-11-07T12:42:23,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:23,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-07T12:42:23,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-07T12:42:23,708 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-07T12:42:23,708 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3200 sec 2024-11-07T12:42:23,710 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.3330 sec 2024-11-07T12:42:23,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 8dc13886c19c8110f075e43282960321 2024-11-07T12:42:23,935 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8dc13886c19c8110f075e43282960321 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-07T12:42:23,937 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1730983343291, max timestamp=1730983343934Num uniques 859; 2024-11-07T12:42:23,937 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1730983343291, max timestamp=1730983343934Num uniques 859; 2024-11-07T12:42:23,937 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1730983343291, max timestamp=1730983343934Num uniques 859; 2024-11-07T12:42:23,937 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-07T12:42:23,937 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-07T12:42:23,937 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-07T12:42:23,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=A 2024-11-07T12:42:23,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:42:23,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=B 2024-11-07T12:42:23,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:42:23,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=C 2024-11-07T12:42:23,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:42:23,943 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/417f7434e70d4e8f9f5180d203e328ea is 50, key is test_row_0/A:col10/1730983343934/Put/seqid=0 2024-11-07T12:42:23,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742174_1350 (size=12301) 2024-11-07T12:42:23,967 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1730983343949, max timestamp=1730983343964Num uniques 859; 2024-11-07T12:42:23,967 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1730983343949, max timestamp=1730983343964Num uniques 859; 2024-11-07T12:42:23,967 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-07T12:42:23,968 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-07T12:42:23,968 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1730983343949, max timestamp=1730983343964Num uniques 859; 2024-11-07T12:42:23,968 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-07T12:42:24,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:24,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42070 deadline: 1730983404030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:24,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:24,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42026 deadline: 1730983404031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:24,042 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:24,042 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:24,042 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:24,043 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:24,043 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:24,043 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:24,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:24,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42034 deadline: 1730983404040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:24,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:24,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:24,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:24,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:24,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42054 deadline: 1730983404042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:24,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:24,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:24,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:24,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:24,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41976 deadline: 1730983404042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:24,056 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:24,057 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:24,057 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:24,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:24,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42070 deadline: 1730983404151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:24,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:24,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42026 deadline: 1730983404151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:24,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:24,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:24,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:24,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:24,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:24,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:24,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:24,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42034 deadline: 1730983404163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:24,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:24,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:24,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:24,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41976 deadline: 1730983404164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:24,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:24,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:24,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42054 deadline: 1730983404166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:24,170 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:24,170 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:24,170 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:24,170 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:24,170 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:24,170 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:24,357 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=467 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/417f7434e70d4e8f9f5180d203e328ea 2024-11-07T12:42:24,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:24,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42026 deadline: 1730983404363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:24,366 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:24,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:24,366 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:24,366 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:24,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42070 deadline: 1730983404363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:24,366 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:24,366 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:24,366 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:24,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:24,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42054 deadline: 1730983404371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:24,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:24,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42034 deadline: 1730983404373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:24,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:24,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:24,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:24,381 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:24,381 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:24,381 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:24,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:24,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41976 deadline: 1730983404382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:24,393 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:24,393 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:24,393 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:24,403 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/25ced5386c5548379c750fa1f2de7c49 is 50, key is test_row_0/B:col10/1730983343934/Put/seqid=0 2024-11-07T12:42:24,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742175_1351 (size=12301) 2024-11-07T12:42:24,433 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=467 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/25ced5386c5548379c750fa1f2de7c49 2024-11-07T12:42:24,454 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/fcf19faee5644bf19ae27396b5cb7bde is 50, key is test_row_0/C:col10/1730983343934/Put/seqid=0 2024-11-07T12:42:24,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742176_1352 (size=12301) 2024-11-07T12:42:24,497 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=467 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/fcf19faee5644bf19ae27396b5cb7bde 2024-11-07T12:42:24,506 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/417f7434e70d4e8f9f5180d203e328ea as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/417f7434e70d4e8f9f5180d203e328ea 2024-11-07T12:42:24,521 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/417f7434e70d4e8f9f5180d203e328ea, entries=150, sequenceid=467, filesize=12.0 K 2024-11-07T12:42:24,521 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/25ced5386c5548379c750fa1f2de7c49 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/25ced5386c5548379c750fa1f2de7c49 2024-11-07T12:42:24,526 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/25ced5386c5548379c750fa1f2de7c49, entries=150, sequenceid=467, filesize=12.0 K 2024-11-07T12:42:24,526 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/fcf19faee5644bf19ae27396b5cb7bde as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/fcf19faee5644bf19ae27396b5cb7bde 2024-11-07T12:42:24,531 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/fcf19faee5644bf19ae27396b5cb7bde, entries=150, sequenceid=467, filesize=12.0 K 2024-11-07T12:42:24,532 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~86.11 KB/88176, currentSize=154.31 KB/158010 for 8dc13886c19c8110f075e43282960321 in 597ms, sequenceid=467, compaction requested=true 2024-11-07T12:42:24,532 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8dc13886c19c8110f075e43282960321: 2024-11-07T12:42:24,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8dc13886c19c8110f075e43282960321:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:42:24,533 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:42:24,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:42:24,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8dc13886c19c8110f075e43282960321:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:42:24,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:42:24,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8dc13886c19c8110f075e43282960321:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:42:24,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:42:24,533 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:42:24,535 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38027 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:42:24,535 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 8dc13886c19c8110f075e43282960321/B is initiating minor compaction (all files) 2024-11-07T12:42:24,535 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8dc13886c19c8110f075e43282960321/B in TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:24,535 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/6f89040571cb4bda9aab6eb7e8d461d3, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/c4e1484e0ef34006bcb6c892ea7ebd0b, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/25ced5386c5548379c750fa1f2de7c49] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp, totalSize=37.1 K 2024-11-07T12:42:24,535 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f89040571cb4bda9aab6eb7e8d461d3, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1730983340890 2024-11-07T12:42:24,536 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting c4e1484e0ef34006bcb6c892ea7ebd0b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=442, earliestPutTs=1730983342061 2024-11-07T12:42:24,536 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38027 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:42:24,536 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): 8dc13886c19c8110f075e43282960321/A is initiating minor compaction (all files) 2024-11-07T12:42:24,536 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 25ced5386c5548379c750fa1f2de7c49, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=467, earliestPutTs=1730983343291 2024-11-07T12:42:24,536 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8dc13886c19c8110f075e43282960321/A in TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:24,536 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/0949ef2b947e4f63900de72628f3a73f, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/25e5e7eacdf04a62a55257ec1ab749ac, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/417f7434e70d4e8f9f5180d203e328ea] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp, totalSize=37.1 K 2024-11-07T12:42:24,536 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0949ef2b947e4f63900de72628f3a73f, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1730983340890 2024-11-07T12:42:24,537 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25e5e7eacdf04a62a55257ec1ab749ac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=442, earliestPutTs=1730983342061 2024-11-07T12:42:24,537 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 417f7434e70d4e8f9f5180d203e328ea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=467, earliestPutTs=1730983343291 2024-11-07T12:42:24,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-07T12:42:24,542 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:42:24,543 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:42:24,546 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8dc13886c19c8110f075e43282960321#B#compaction#303 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:42:24,547 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/bf219f9fdef7487884ff792c793c831e is 50, key is test_row_0/B:col10/1730983343934/Put/seqid=0 2024-11-07T12:42:24,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-07T12:42:24,549 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:42:24,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-07T12:42:24,551 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:42:24,551 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:42:24,563 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8dc13886c19c8110f075e43282960321#A#compaction#304 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:42:24,563 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/d67f5b4de2c44e3ba7fd66d65509e95b is 50, key is test_row_0/A:col10/1730983343934/Put/seqid=0 2024-11-07T12:42:24,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742177_1353 (size=13527) 2024-11-07T12:42:24,626 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/bf219f9fdef7487884ff792c793c831e as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/bf219f9fdef7487884ff792c793c831e 2024-11-07T12:42:24,637 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8dc13886c19c8110f075e43282960321/B of 8dc13886c19c8110f075e43282960321 into bf219f9fdef7487884ff792c793c831e(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:42:24,637 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8dc13886c19c8110f075e43282960321: 2024-11-07T12:42:24,637 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., storeName=8dc13886c19c8110f075e43282960321/B, priority=13, startTime=1730983344533; duration=0sec 2024-11-07T12:42:24,638 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:42:24,638 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8dc13886c19c8110f075e43282960321:B 2024-11-07T12:42:24,638 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:42:24,639 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38027 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:42:24,639 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 8dc13886c19c8110f075e43282960321/C is initiating minor compaction (all files) 2024-11-07T12:42:24,639 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8dc13886c19c8110f075e43282960321/C in TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:24,639 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/29484e2d8a874546958567a53a29b7ec, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/04470e32f61a4f56b54a9c7e5b655ac8, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/fcf19faee5644bf19ae27396b5cb7bde] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp, totalSize=37.1 K 2024-11-07T12:42:24,640 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 29484e2d8a874546958567a53a29b7ec, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1730983340890 2024-11-07T12:42:24,641 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 04470e32f61a4f56b54a9c7e5b655ac8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=442, earliestPutTs=1730983342061 2024-11-07T12:42:24,641 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting fcf19faee5644bf19ae27396b5cb7bde, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=467, earliestPutTs=1730983343291 2024-11-07T12:42:24,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742178_1354 (size=13527) 2024-11-07T12:42:24,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-07T12:42:24,659 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8dc13886c19c8110f075e43282960321#C#compaction#305 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:42:24,659 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/75038c1441f04469921acf3a4ac1da42 is 50, key is test_row_0/C:col10/1730983343934/Put/seqid=0 2024-11-07T12:42:24,660 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/d67f5b4de2c44e3ba7fd66d65509e95b as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/d67f5b4de2c44e3ba7fd66d65509e95b 2024-11-07T12:42:24,666 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8dc13886c19c8110f075e43282960321/A of 8dc13886c19c8110f075e43282960321 into d67f5b4de2c44e3ba7fd66d65509e95b(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:42:24,666 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8dc13886c19c8110f075e43282960321: 2024-11-07T12:42:24,666 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., storeName=8dc13886c19c8110f075e43282960321/A, priority=13, startTime=1730983344532; duration=0sec 2024-11-07T12:42:24,666 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:42:24,666 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8dc13886c19c8110f075e43282960321:A 2024-11-07T12:42:24,675 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8dc13886c19c8110f075e43282960321 3/3 column families, dataSize=161.02 KB heapSize=451.31 KB 2024-11-07T12:42:24,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=A 2024-11-07T12:42:24,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-07T12:42:24,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=B 2024-11-07T12:42:24,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-07T12:42:24,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=C 2024-11-07T12:42:24,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-07T12:42:24,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 8dc13886c19c8110f075e43282960321 2024-11-07T12:42:24,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742179_1355 (size=13527) 2024-11-07T12:42:24,689 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/747cdf3fd7aa4bce95ee27322da9fa6d is 50, key is test_row_0/A:col10/1730983344037/Put/seqid=0 2024-11-07T12:42:24,697 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/75038c1441f04469921acf3a4ac1da42 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/75038c1441f04469921acf3a4ac1da42 2024-11-07T12:42:24,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:24,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42054 deadline: 1730983404692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:24,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:24,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:24,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:24,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42034 deadline: 1730983404692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:24,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:24,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:24,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42026 deadline: 1730983404693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:24,700 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:24,700 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:24,700 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:24,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:24,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:24,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:24,703 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8dc13886c19c8110f075e43282960321/C of 8dc13886c19c8110f075e43282960321 into 75038c1441f04469921acf3a4ac1da42(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:42:24,703 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8dc13886c19c8110f075e43282960321: 2024-11-07T12:42:24,703 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., storeName=8dc13886c19c8110f075e43282960321/C, priority=13, startTime=1730983344533; duration=0sec 2024-11-07T12:42:24,703 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:42:24,703 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8dc13886c19c8110f075e43282960321:C 2024-11-07T12:42:24,704 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-07T12:42:24,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:24,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. as already flushing 2024-11-07T12:42:24,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:24,705 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:24,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:24,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:24,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:24,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42070 deadline: 1730983404698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:24,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:24,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:24,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:24,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:24,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41976 deadline: 1730983404702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:24,710 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:24,710 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:24,710 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:24,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742180_1356 (size=14741) 2024-11-07T12:42:24,734 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=507 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/747cdf3fd7aa4bce95ee27322da9fa6d 2024-11-07T12:42:24,767 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/d7a160ce7eb54f96a0f27d4723952b9c is 50, key is test_row_0/B:col10/1730983344037/Put/seqid=0 2024-11-07T12:42:24,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742181_1357 (size=12301) 2024-11-07T12:42:24,776 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=507 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/d7a160ce7eb54f96a0f27d4723952b9c 2024-11-07T12:42:24,806 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/a90cc24886e94f02a1a2d95e7abfbd87 is 50, key is test_row_0/C:col10/1730983344037/Put/seqid=0 2024-11-07T12:42:24,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:24,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42026 deadline: 1730983404812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:24,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:24,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:24,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:24,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:24,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42070 deadline: 1730983404812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:24,817 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:24,817 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:24,817 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:24,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742182_1358 (size=12301) 2024-11-07T12:42:24,860 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-07T12:42:24,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:24,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. as already flushing 2024-11-07T12:42:24,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:24,860 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:24,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:24,860 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=507 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/a90cc24886e94f02a1a2d95e7abfbd87 2024-11-07T12:42:24,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:24,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-07T12:42:24,874 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/747cdf3fd7aa4bce95ee27322da9fa6d as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/747cdf3fd7aa4bce95ee27322da9fa6d 2024-11-07T12:42:24,882 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/747cdf3fd7aa4bce95ee27322da9fa6d, entries=200, sequenceid=507, filesize=14.4 K 2024-11-07T12:42:24,883 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/d7a160ce7eb54f96a0f27d4723952b9c as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/d7a160ce7eb54f96a0f27d4723952b9c 2024-11-07T12:42:24,890 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/d7a160ce7eb54f96a0f27d4723952b9c, entries=150, sequenceid=507, filesize=12.0 K 2024-11-07T12:42:24,891 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/a90cc24886e94f02a1a2d95e7abfbd87 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/a90cc24886e94f02a1a2d95e7abfbd87 2024-11-07T12:42:24,898 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/a90cc24886e94f02a1a2d95e7abfbd87, entries=150, sequenceid=507, filesize=12.0 K 2024-11-07T12:42:24,899 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~451.27 KB/462096, currentSize=26.84 KB/27480 for 8dc13886c19c8110f075e43282960321 in 224ms, sequenceid=507, compaction requested=false 2024-11-07T12:42:24,899 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8dc13886c19c8110f075e43282960321: 2024-11-07T12:42:25,014 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-07T12:42:25,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:25,015 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing 8dc13886c19c8110f075e43282960321 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-07T12:42:25,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=A 2024-11-07T12:42:25,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:25,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=B 2024-11-07T12:42:25,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:25,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=C 2024-11-07T12:42:25,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:25,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/4f6f9d89aeae4be99adc7124f96bcc23 is 50, key is test_row_0/A:col10/1730983344696/Put/seqid=0 2024-11-07T12:42:25,032 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. as already flushing 2024-11-07T12:42:25,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 8dc13886c19c8110f075e43282960321 2024-11-07T12:42:25,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742183_1359 (size=9857) 2024-11-07T12:42:25,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:25,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42070 deadline: 1730983405093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:25,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:25,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42026 deadline: 1730983405094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:25,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:25,099 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:25,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:25,099 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:25,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:25,099 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:25,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-07T12:42:25,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:25,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42070 deadline: 1730983405203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:25,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:25,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42026 deadline: 1730983405203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:25,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:25,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:25,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:25,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:25,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:25,209 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:25,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:25,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42034 deadline: 1730983405211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:25,215 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:25,215 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:25,215 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:25,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:25,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42054 deadline: 1730983405211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:25,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:25,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:25,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:25,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:25,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41976 deadline: 1730983405222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:25,227 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:25,227 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:25,227 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:25,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:25,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42026 deadline: 1730983405412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:25,414 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:25,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:25,414 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:25,414 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:25,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42070 deadline: 1730983405412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:25,414 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:25,414 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:25,414 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:25,461 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=515 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/4f6f9d89aeae4be99adc7124f96bcc23 2024-11-07T12:42:25,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/a1ff26a2b56346938b518881f6bea3ac is 50, key is test_row_0/B:col10/1730983344696/Put/seqid=0 2024-11-07T12:42:25,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742184_1360 (size=9857) 2024-11-07T12:42:25,488 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=515 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/a1ff26a2b56346938b518881f6bea3ac 2024-11-07T12:42:25,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/5a441990bc0443658fdfb7d47a77f160 is 50, key is test_row_0/C:col10/1730983344696/Put/seqid=0 2024-11-07T12:42:25,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742185_1361 (size=9857) 2024-11-07T12:42:25,514 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=515 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/5a441990bc0443658fdfb7d47a77f160 2024-11-07T12:42:25,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/4f6f9d89aeae4be99adc7124f96bcc23 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/4f6f9d89aeae4be99adc7124f96bcc23 2024-11-07T12:42:25,563 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/4f6f9d89aeae4be99adc7124f96bcc23, entries=100, sequenceid=515, filesize=9.6 K 2024-11-07T12:42:25,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/a1ff26a2b56346938b518881f6bea3ac as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/a1ff26a2b56346938b518881f6bea3ac 2024-11-07T12:42:25,568 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/a1ff26a2b56346938b518881f6bea3ac, entries=100, sequenceid=515, filesize=9.6 K 2024-11-07T12:42:25,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/5a441990bc0443658fdfb7d47a77f160 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/5a441990bc0443658fdfb7d47a77f160 2024-11-07T12:42:25,573 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/5a441990bc0443658fdfb7d47a77f160, entries=100, sequenceid=515, filesize=9.6 K 2024-11-07T12:42:25,574 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=147.60 KB/151140 for 8dc13886c19c8110f075e43282960321 in 559ms, sequenceid=515, compaction requested=true 2024-11-07T12:42:25,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for 8dc13886c19c8110f075e43282960321: 2024-11-07T12:42:25,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:25,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-07T12:42:25,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-07T12:42:25,578 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-07T12:42:25,578 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0250 sec 2024-11-07T12:42:25,580 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 1.0360 sec 2024-11-07T12:42:25,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-07T12:42:25,682 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:42:25,683 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:42:25,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-11-07T12:42:25,685 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:42:25,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-07T12:42:25,688 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:42:25,688 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:42:25,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 8dc13886c19c8110f075e43282960321 2024-11-07T12:42:25,725 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8dc13886c19c8110f075e43282960321 3/3 column families, dataSize=154.31 KB heapSize=459.66 KB 2024-11-07T12:42:25,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=A 2024-11-07T12:42:25,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:42:25,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=B 2024-11-07T12:42:25,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:42:25,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=C 2024-11-07T12:42:25,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:42:25,733 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/95c60ea6737e411b942c0f990adff55c is 50, key is test_row_0/A:col10/1730983345723/Put/seqid=0 2024-11-07T12:42:25,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:25,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42026 deadline: 1730983405735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:25,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:25,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:25,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:25,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:25,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42070 deadline: 1730983405741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:25,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:25,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:25,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:25,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742186_1362 (size=12301) 2024-11-07T12:42:25,759 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=541 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/95c60ea6737e411b942c0f990adff55c 2024-11-07T12:42:25,771 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/683dfcf3088042038ea16c62f1bb63ad is 50, key is test_row_0/B:col10/1730983345723/Put/seqid=0 2024-11-07T12:42:25,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-07T12:42:25,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742187_1363 (size=12301) 2024-11-07T12:42:25,814 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=541 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/683dfcf3088042038ea16c62f1bb63ad 2024-11-07T12:42:25,831 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/66c7c195b2ed4884a0ff3a003f9a1508 is 50, key is test_row_0/C:col10/1730983345723/Put/seqid=0 2024-11-07T12:42:25,841 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-07T12:42:25,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:25,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. as already flushing 2024-11-07T12:42:25,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:25,842 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:25,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:25,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:25,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:25,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42026 deadline: 1730983405843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:25,845 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:25,845 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:25,845 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:25,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:25,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42070 deadline: 1730983405852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:25,858 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:25,858 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:25,858 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:25,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742188_1364 (size=12301) 2024-11-07T12:42:25,878 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=541 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/66c7c195b2ed4884a0ff3a003f9a1508 2024-11-07T12:42:25,887 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/95c60ea6737e411b942c0f990adff55c as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/95c60ea6737e411b942c0f990adff55c 2024-11-07T12:42:25,892 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/95c60ea6737e411b942c0f990adff55c, entries=150, sequenceid=541, filesize=12.0 K 2024-11-07T12:42:25,892 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/683dfcf3088042038ea16c62f1bb63ad as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/683dfcf3088042038ea16c62f1bb63ad 2024-11-07T12:42:25,899 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/683dfcf3088042038ea16c62f1bb63ad, entries=150, sequenceid=541, filesize=12.0 K 2024-11-07T12:42:25,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/66c7c195b2ed4884a0ff3a003f9a1508 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/66c7c195b2ed4884a0ff3a003f9a1508 2024-11-07T12:42:25,906 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/66c7c195b2ed4884a0ff3a003f9a1508, entries=150, sequenceid=541, filesize=12.0 K 2024-11-07T12:42:25,907 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~459.61 KB/470640, currentSize=20.13 KB/20610 for 8dc13886c19c8110f075e43282960321 in 182ms, sequenceid=541, compaction requested=true 2024-11-07T12:42:25,907 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8dc13886c19c8110f075e43282960321: 2024-11-07T12:42:25,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8dc13886c19c8110f075e43282960321:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:42:25,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:42:25,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8dc13886c19c8110f075e43282960321:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:42:25,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-07T12:42:25,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8dc13886c19c8110f075e43282960321:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:42:25,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-07T12:42:25,908 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:42:25,908 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:42:25,913 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47986 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:42:25,914 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 8dc13886c19c8110f075e43282960321/C is initiating minor compaction (all files) 2024-11-07T12:42:25,914 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8dc13886c19c8110f075e43282960321/C in TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:25,914 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/75038c1441f04469921acf3a4ac1da42, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/a90cc24886e94f02a1a2d95e7abfbd87, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/5a441990bc0443658fdfb7d47a77f160, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/66c7c195b2ed4884a0ff3a003f9a1508] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp, totalSize=46.9 K 2024-11-07T12:42:25,914 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50426 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:42:25,914 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): 8dc13886c19c8110f075e43282960321/A is initiating minor compaction (all files) 2024-11-07T12:42:25,914 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8dc13886c19c8110f075e43282960321/A in TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:25,914 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/d67f5b4de2c44e3ba7fd66d65509e95b, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/747cdf3fd7aa4bce95ee27322da9fa6d, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/4f6f9d89aeae4be99adc7124f96bcc23, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/95c60ea6737e411b942c0f990adff55c] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp, totalSize=49.2 K 2024-11-07T12:42:25,915 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting d67f5b4de2c44e3ba7fd66d65509e95b, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=467, earliestPutTs=1730983343291 2024-11-07T12:42:25,915 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 75038c1441f04469921acf3a4ac1da42, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=467, earliestPutTs=1730983343291 2024-11-07T12:42:25,916 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting a90cc24886e94f02a1a2d95e7abfbd87, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=507, earliestPutTs=1730983344028 2024-11-07T12:42:25,916 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 747cdf3fd7aa4bce95ee27322da9fa6d, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=507, earliestPutTs=1730983344028 2024-11-07T12:42:25,916 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a441990bc0443658fdfb7d47a77f160, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=515, earliestPutTs=1730983344687 2024-11-07T12:42:25,917 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f6f9d89aeae4be99adc7124f96bcc23, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=515, earliestPutTs=1730983344687 2024-11-07T12:42:25,918 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95c60ea6737e411b942c0f990adff55c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=541, earliestPutTs=1730983345083 2024-11-07T12:42:25,919 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 66c7c195b2ed4884a0ff3a003f9a1508, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=541, earliestPutTs=1730983345083 2024-11-07T12:42:25,939 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8dc13886c19c8110f075e43282960321#C#compaction#315 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:42:25,940 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/a536f9cc65894897be4709b941497bf4 is 50, key is test_row_0/C:col10/1730983345723/Put/seqid=0 2024-11-07T12:42:25,945 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8dc13886c19c8110f075e43282960321#A#compaction#316 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:42:25,945 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/ae2cf5bebcec4934959cd90d334afd4f is 50, key is test_row_0/A:col10/1730983345723/Put/seqid=0 2024-11-07T12:42:25,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742189_1365 (size=13663) 2024-11-07T12:42:25,990 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/ae2cf5bebcec4934959cd90d334afd4f as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/ae2cf5bebcec4934959cd90d334afd4f 2024-11-07T12:42:25,996 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-07T12:42:25,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:25,996 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing 8dc13886c19c8110f075e43282960321 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-07T12:42:25,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=A 2024-11-07T12:42:25,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:25,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=B 2024-11-07T12:42:25,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:25,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=C 2024-11-07T12:42:25,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:25,999 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8dc13886c19c8110f075e43282960321/A of 8dc13886c19c8110f075e43282960321 into ae2cf5bebcec4934959cd90d334afd4f(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:42:25,999 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8dc13886c19c8110f075e43282960321: 2024-11-07T12:42:25,999 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., storeName=8dc13886c19c8110f075e43282960321/A, priority=12, startTime=1730983345908; duration=0sec 2024-11-07T12:42:26,000 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:42:26,000 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8dc13886c19c8110f075e43282960321:A 2024-11-07T12:42:26,000 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:42:26,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-07T12:42:26,002 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47986 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:42:26,002 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): 8dc13886c19c8110f075e43282960321/B is initiating minor compaction (all files) 2024-11-07T12:42:26,002 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8dc13886c19c8110f075e43282960321/B in TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:26,003 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/bf219f9fdef7487884ff792c793c831e, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/d7a160ce7eb54f96a0f27d4723952b9c, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/a1ff26a2b56346938b518881f6bea3ac, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/683dfcf3088042038ea16c62f1bb63ad] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp, totalSize=46.9 K 2024-11-07T12:42:26,003 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf219f9fdef7487884ff792c793c831e, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=467, earliestPutTs=1730983343291 2024-11-07T12:42:26,003 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7a160ce7eb54f96a0f27d4723952b9c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=507, earliestPutTs=1730983344028 2024-11-07T12:42:26,003 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1ff26a2b56346938b518881f6bea3ac, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=515, earliestPutTs=1730983344687 2024-11-07T12:42:26,004 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 683dfcf3088042038ea16c62f1bb63ad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=541, earliestPutTs=1730983345083 2024-11-07T12:42:26,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742190_1366 (size=13663) 2024-11-07T12:42:26,011 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/a536f9cc65894897be4709b941497bf4 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/a536f9cc65894897be4709b941497bf4 2024-11-07T12:42:26,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/21e737d8256c48968b0bb2535df3841f is 50, key is test_row_0/A:col10/1730983345734/Put/seqid=0 2024-11-07T12:42:26,020 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8dc13886c19c8110f075e43282960321/C of 8dc13886c19c8110f075e43282960321 into a536f9cc65894897be4709b941497bf4(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:42:26,020 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8dc13886c19c8110f075e43282960321: 2024-11-07T12:42:26,020 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., storeName=8dc13886c19c8110f075e43282960321/C, priority=12, startTime=1730983345908; duration=0sec 2024-11-07T12:42:26,020 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:42:26,020 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8dc13886c19c8110f075e43282960321:C 2024-11-07T12:42:26,036 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8dc13886c19c8110f075e43282960321#B#compaction#318 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:42:26,036 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/3e3a08ca55a34fd4a1397964f64de38c is 50, key is test_row_0/B:col10/1730983345723/Put/seqid=0 2024-11-07T12:42:26,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742191_1367 (size=12301) 2024-11-07T12:42:26,043 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=547 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/21e737d8256c48968b0bb2535df3841f 2024-11-07T12:42:26,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742192_1368 (size=13663) 2024-11-07T12:42:26,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/a86aa6d7422448028405cbd45db67ff6 is 50, key is test_row_0/B:col10/1730983345734/Put/seqid=0 2024-11-07T12:42:26,062 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/3e3a08ca55a34fd4a1397964f64de38c as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/3e3a08ca55a34fd4a1397964f64de38c 2024-11-07T12:42:26,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 8dc13886c19c8110f075e43282960321 2024-11-07T12:42:26,068 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8dc13886c19c8110f075e43282960321/B of 8dc13886c19c8110f075e43282960321 into 3e3a08ca55a34fd4a1397964f64de38c(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:42:26,068 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8dc13886c19c8110f075e43282960321: 2024-11-07T12:42:26,068 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., storeName=8dc13886c19c8110f075e43282960321/B, priority=12, startTime=1730983345908; duration=0sec 2024-11-07T12:42:26,068 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:42:26,068 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8dc13886c19c8110f075e43282960321:B 2024-11-07T12:42:26,068 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. as already flushing 2024-11-07T12:42:26,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742193_1369 (size=12301) 2024-11-07T12:42:26,078 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=547 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/a86aa6d7422448028405cbd45db67ff6 2024-11-07T12:42:26,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/3027b533ce53416d9a813df6bba1e9c4 is 50, key is test_row_0/C:col10/1730983345734/Put/seqid=0 2024-11-07T12:42:26,114 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1730983346092, max timestamp=1730983346110Num uniques 859; 2024-11-07T12:42:26,114 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1730983346092, max timestamp=1730983346110Num uniques 859; 2024-11-07T12:42:26,114 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-07T12:42:26,114 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-07T12:42:26,114 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1730983346092, max timestamp=1730983346110Num uniques 859; 2024-11-07T12:42:26,114 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-07T12:42:26,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742194_1370 (size=12301) 2024-11-07T12:42:26,142 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=547 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/3027b533ce53416d9a813df6bba1e9c4 2024-11-07T12:42:26,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/21e737d8256c48968b0bb2535df3841f as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/21e737d8256c48968b0bb2535df3841f 2024-11-07T12:42:26,159 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/21e737d8256c48968b0bb2535df3841f, entries=150, sequenceid=547, filesize=12.0 K 2024-11-07T12:42:26,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/a86aa6d7422448028405cbd45db67ff6 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/a86aa6d7422448028405cbd45db67ff6 2024-11-07T12:42:26,172 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/a86aa6d7422448028405cbd45db67ff6, entries=150, sequenceid=547, filesize=12.0 K 2024-11-07T12:42:26,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/3027b533ce53416d9a813df6bba1e9c4 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/3027b533ce53416d9a813df6bba1e9c4 2024-11-07T12:42:26,177 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/3027b533ce53416d9a813df6bba1e9c4, entries=150, sequenceid=547, filesize=12.0 K 2024-11-07T12:42:26,178 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=127.47 KB/130530 for 8dc13886c19c8110f075e43282960321 in 182ms, sequenceid=547, compaction requested=false 2024-11-07T12:42:26,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for 8dc13886c19c8110f075e43282960321: 2024-11-07T12:42:26,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:26,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-11-07T12:42:26,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-11-07T12:42:26,181 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-07T12:42:26,181 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 491 msec 2024-11-07T12:42:26,184 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 499 msec 2024-11-07T12:42:26,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 8dc13886c19c8110f075e43282960321 2024-11-07T12:42:26,185 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8dc13886c19c8110f075e43282960321 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-07T12:42:26,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=A 2024-11-07T12:42:26,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:42:26,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=B 2024-11-07T12:42:26,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:42:26,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=C 2024-11-07T12:42:26,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:42:26,196 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/bb308c9e0a14413cb3f2c684498a6c8e is 50, key is test_row_0/A:col10/1730983346183/Put/seqid=0 2024-11-07T12:42:26,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:26,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42070 deadline: 1730983406214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:26,222 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:26,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:26,222 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:26,222 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:26,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42026 deadline: 1730983406221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:26,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:26,229 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:26,229 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:26,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:26,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42034 deadline: 1730983406222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:26,230 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to db9ad1cb6cf9,34293,1730983260217 for 'test_row_2' in 8dc13886c19c8110f075e43282960321 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:42:26,231 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:26,231 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:26,231 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:26,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:26,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42054 deadline: 1730983406233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:26,239 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to db9ad1cb6cf9,34293,1730983260217 for 'test_row_2' in 8dc13886c19c8110f075e43282960321 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:42:26,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:26,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:26,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:26,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:26,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41976 deadline: 1730983406242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:26,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742195_1371 (size=17181) 2024-11-07T12:42:26,255 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to db9ad1cb6cf9,34293,1730983260217 for 'test_row_0' in 8dc13886c19c8110f075e43282960321 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:42:26,256 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:26,256 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:26,256 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:26,257 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=587 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/bb308c9e0a14413cb3f2c684498a6c8e 2024-11-07T12:42:26,307 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/520d19ca1b70429d815aea427a2c5789 is 50, key is test_row_0/B:col10/1730983346183/Put/seqid=0 2024-11-07T12:42:26,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-07T12:42:26,325 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:42:26,325 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:42:26,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees 2024-11-07T12:42:26,327 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:42:26,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-07T12:42:26,329 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:42:26,329 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:42:26,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:26,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42070 deadline: 1730983406331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:26,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:26,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42026 deadline: 1730983406333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:26,339 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:26,339 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:26,339 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:26,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:26,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:26,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:26,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742196_1372 (size=12301) 2024-11-07T12:42:26,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-07T12:42:26,486 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-07T12:42:26,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:26,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. as already flushing 2024-11-07T12:42:26,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:26,487 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:26,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:26,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:26,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:26,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42070 deadline: 1730983406549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:26,553 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:26,553 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:26,553 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:26,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:26,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42026 deadline: 1730983406552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:26,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:26,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:26,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:26,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-07T12:42:26,652 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-07T12:42:26,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:26,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. as already flushing 2024-11-07T12:42:26,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:26,653 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:26,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:26,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:26,747 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=587 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/520d19ca1b70429d815aea427a2c5789 2024-11-07T12:42:26,813 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-07T12:42:26,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:26,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. as already flushing 2024-11-07T12:42:26,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:26,814 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:26,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:26,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:26,825 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/19e1713981fd4d5da60d3b5cb6f983dc is 50, key is test_row_0/C:col10/1730983346183/Put/seqid=0 2024-11-07T12:42:26,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742197_1373 (size=12301) 2024-11-07T12:42:26,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:26,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42070 deadline: 1730983406861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:26,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:26,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42026 deadline: 1730983406862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:26,868 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:26,868 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:26,868 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:26,868 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:26,868 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:26,868 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:42:26,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-07T12:42:26,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-07T12:42:26,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:26,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. as already flushing 2024-11-07T12:42:26,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:26,973 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:26,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:26,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:27,132 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-07T12:42:27,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:27,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. as already flushing 2024-11-07T12:42:27,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:27,134 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:27,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:27,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:27,260 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=587 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/19e1713981fd4d5da60d3b5cb6f983dc 2024-11-07T12:42:27,291 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-07T12:42:27,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:27,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. as already flushing 2024-11-07T12:42:27,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:27,292 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:27,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:27,293 INFO [Thread-1150 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1150. 2024-11-07T12:42:27,293 DEBUG [Thread-1150 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:42:27,293 DEBUG [Thread-1150 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:42:27,293 DEBUG [Thread-1150 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:42:27,294 INFO [Thread-1154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1154. 2024-11-07T12:42:27,294 DEBUG [Thread-1154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:42:27,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:27,294 DEBUG [Thread-1154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:42:27,294 DEBUG [Thread-1154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:42:27,294 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:42:27,296 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:42:27,297 INFO [Thread-1156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1156. 2024-11-07T12:42:27,297 DEBUG [Thread-1156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:42:27,297 DEBUG [Thread-1156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:42:27,298 DEBUG [Thread-1156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:42:27,299 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:42:27,300 INFO [Thread-1152 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1152. 2024-11-07T12:42:27,300 DEBUG [Thread-1152 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:42:27,300 DEBUG [Thread-1152 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:42:27,300 DEBUG [Thread-1152 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:42:27,300 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:42:27,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/bb308c9e0a14413cb3f2c684498a6c8e as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/bb308c9e0a14413cb3f2c684498a6c8e 2024-11-07T12:42:27,302 INFO [Thread-1148 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1148. 2024-11-07T12:42:27,302 DEBUG [Thread-1148 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:42:27,302 DEBUG [Thread-1148 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:42:27,302 DEBUG [Thread-1148 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:42:27,304 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:42:27,309 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/bb308c9e0a14413cb3f2c684498a6c8e, entries=250, sequenceid=587, filesize=16.8 K 2024-11-07T12:42:27,312 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/520d19ca1b70429d815aea427a2c5789 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/520d19ca1b70429d815aea427a2c5789 2024-11-07T12:42:27,318 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/520d19ca1b70429d815aea427a2c5789, entries=150, sequenceid=587, filesize=12.0 K 2024-11-07T12:42:27,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/19e1713981fd4d5da60d3b5cb6f983dc as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/19e1713981fd4d5da60d3b5cb6f983dc 2024-11-07T12:42:27,327 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/19e1713981fd4d5da60d3b5cb6f983dc, entries=150, sequenceid=587, filesize=12.0 K 2024-11-07T12:42:27,329 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 8dc13886c19c8110f075e43282960321 in 1144ms, sequenceid=587, compaction requested=true 2024-11-07T12:42:27,329 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8dc13886c19c8110f075e43282960321: 2024-11-07T12:42:27,329 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:42:27,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8dc13886c19c8110f075e43282960321:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:42:27,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:42:27,329 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:42:27,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8dc13886c19c8110f075e43282960321:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:42:27,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:42:27,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8dc13886c19c8110f075e43282960321:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:42:27,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:42:27,332 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38265 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:42:27,332 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43145 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:42:27,332 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 8dc13886c19c8110f075e43282960321/B is initiating minor compaction (all files) 2024-11-07T12:42:27,332 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): 8dc13886c19c8110f075e43282960321/A is initiating minor compaction (all files) 2024-11-07T12:42:27,332 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8dc13886c19c8110f075e43282960321/B in TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:27,332 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8dc13886c19c8110f075e43282960321/A in TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:27,332 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/3e3a08ca55a34fd4a1397964f64de38c, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/a86aa6d7422448028405cbd45db67ff6, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/520d19ca1b70429d815aea427a2c5789] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp, totalSize=37.4 K 2024-11-07T12:42:27,332 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/ae2cf5bebcec4934959cd90d334afd4f, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/21e737d8256c48968b0bb2535df3841f, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/bb308c9e0a14413cb3f2c684498a6c8e] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp, totalSize=42.1 K 2024-11-07T12:42:27,333 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e3a08ca55a34fd4a1397964f64de38c, keycount=150, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=541, earliestPutTs=1730983345083 2024-11-07T12:42:27,333 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting a86aa6d7422448028405cbd45db67ff6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=547, earliestPutTs=1730983345726 2024-11-07T12:42:27,333 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae2cf5bebcec4934959cd90d334afd4f, keycount=150, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=541, earliestPutTs=1730983345083 2024-11-07T12:42:27,333 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21e737d8256c48968b0bb2535df3841f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=547, earliestPutTs=1730983345726 2024-11-07T12:42:27,333 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 520d19ca1b70429d815aea427a2c5789, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=587, earliestPutTs=1730983346175 2024-11-07T12:42:27,334 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb308c9e0a14413cb3f2c684498a6c8e, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=587, earliestPutTs=1730983346133 2024-11-07T12:42:27,350 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8dc13886c19c8110f075e43282960321#B#compaction#324 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:42:27,351 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/9a21fdc5f989431a8503b0895c21dd43 is 50, key is test_row_0/B:col10/1730983346183/Put/seqid=0 2024-11-07T12:42:27,359 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8dc13886c19c8110f075e43282960321#A#compaction#325 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:42:27,360 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/943794d6132d46bb8d9a7828dc9a431b is 50, key is test_row_0/A:col10/1730983346183/Put/seqid=0 2024-11-07T12:42:27,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742198_1374 (size=13765) 2024-11-07T12:42:27,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 8dc13886c19c8110f075e43282960321 2024-11-07T12:42:27,402 INFO [Thread-1139 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1139. 2024-11-07T12:42:27,402 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8dc13886c19c8110f075e43282960321 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-07T12:42:27,402 DEBUG [Thread-1139 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:42:27,402 DEBUG [Thread-1139 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:42:27,403 DEBUG [Thread-1139 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:42:27,403 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:42:27,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=A 2024-11-07T12:42:27,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:27,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=B 2024-11-07T12:42:27,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:27,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=C 2024-11-07T12:42:27,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:27,405 INFO [Thread-1141 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1141. 2024-11-07T12:42:27,405 DEBUG [Thread-1141 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:42:27,405 DEBUG [Thread-1141 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:42:27,405 DEBUG [Thread-1141 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:42:27,408 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:42:27,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742199_1375 (size=13765) 2024-11-07T12:42:27,432 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/8a8ab913fbda402abe9d3b5ec6a58994 is 50, key is test_row_0/A:col10/1730983347391/Put/seqid=0 2024-11-07T12:42:27,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742200_1376 (size=12301) 2024-11-07T12:42:27,451 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-07T12:42:27,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:27,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. as already flushing 2024-11-07T12:42:27,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:27,451 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:27,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:27,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:27,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-07T12:42:27,606 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-07T12:42:27,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:27,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. as already flushing 2024-11-07T12:42:27,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:27,607 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:27,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:27,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:27,760 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-07T12:42:27,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:27,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. as already flushing 2024-11-07T12:42:27,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:27,761 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:27,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:27,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:27,796 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/9a21fdc5f989431a8503b0895c21dd43 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/9a21fdc5f989431a8503b0895c21dd43 2024-11-07T12:42:27,801 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8dc13886c19c8110f075e43282960321/B of 8dc13886c19c8110f075e43282960321 into 9a21fdc5f989431a8503b0895c21dd43(size=13.4 K), total size for store is 13.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:42:27,801 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8dc13886c19c8110f075e43282960321: 2024-11-07T12:42:27,801 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., storeName=8dc13886c19c8110f075e43282960321/B, priority=13, startTime=1730983347329; duration=0sec 2024-11-07T12:42:27,801 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:42:27,801 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8dc13886c19c8110f075e43282960321:B 2024-11-07T12:42:27,801 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:42:27,802 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38265 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:42:27,802 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 8dc13886c19c8110f075e43282960321/C is initiating minor compaction (all files) 2024-11-07T12:42:27,802 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8dc13886c19c8110f075e43282960321/C in TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:27,802 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/a536f9cc65894897be4709b941497bf4, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/3027b533ce53416d9a813df6bba1e9c4, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/19e1713981fd4d5da60d3b5cb6f983dc] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp, totalSize=37.4 K 2024-11-07T12:42:27,802 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting a536f9cc65894897be4709b941497bf4, keycount=150, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=541, earliestPutTs=1730983345083 2024-11-07T12:42:27,803 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 3027b533ce53416d9a813df6bba1e9c4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=547, earliestPutTs=1730983345726 2024-11-07T12:42:27,803 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 19e1713981fd4d5da60d3b5cb6f983dc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=587, earliestPutTs=1730983346175 2024-11-07T12:42:27,810 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8dc13886c19c8110f075e43282960321#C#compaction#327 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:42:27,811 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/0ca07d34d37049bd8e51df149abea9bd is 50, key is test_row_0/C:col10/1730983346183/Put/seqid=0 2024-11-07T12:42:27,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742201_1377 (size=13765) 2024-11-07T12:42:27,827 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/943794d6132d46bb8d9a7828dc9a431b as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/943794d6132d46bb8d9a7828dc9a431b 2024-11-07T12:42:27,828 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/0ca07d34d37049bd8e51df149abea9bd as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/0ca07d34d37049bd8e51df149abea9bd 2024-11-07T12:42:27,831 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8dc13886c19c8110f075e43282960321/A of 8dc13886c19c8110f075e43282960321 into 943794d6132d46bb8d9a7828dc9a431b(size=13.4 K), total size for store is 13.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:42:27,831 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8dc13886c19c8110f075e43282960321: 2024-11-07T12:42:27,831 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., storeName=8dc13886c19c8110f075e43282960321/A, priority=13, startTime=1730983347329; duration=0sec 2024-11-07T12:42:27,831 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:42:27,831 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8dc13886c19c8110f075e43282960321:A 2024-11-07T12:42:27,831 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8dc13886c19c8110f075e43282960321/C of 8dc13886c19c8110f075e43282960321 into 0ca07d34d37049bd8e51df149abea9bd(size=13.4 K), total size for store is 13.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:42:27,831 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8dc13886c19c8110f075e43282960321: 2024-11-07T12:42:27,832 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321., storeName=8dc13886c19c8110f075e43282960321/C, priority=13, startTime=1730983347330; duration=0sec 2024-11-07T12:42:27,832 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:42:27,832 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8dc13886c19c8110f075e43282960321:C 2024-11-07T12:42:27,837 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-07T12:42:27,851 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=598 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/8a8ab913fbda402abe9d3b5ec6a58994 2024-11-07T12:42:27,857 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/80a435f2dec24d28b909f0a2f35181c0 is 50, key is test_row_0/B:col10/1730983347391/Put/seqid=0 2024-11-07T12:42:27,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742202_1378 (size=12301) 2024-11-07T12:42:27,914 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-07T12:42:27,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:27,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. as already flushing 2024-11-07T12:42:27,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:27,915 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:27,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:27,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:28,067 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-07T12:42:28,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:28,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. as already flushing 2024-11-07T12:42:28,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:28,068 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:28,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:28,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:28,221 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-07T12:42:28,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:28,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. as already flushing 2024-11-07T12:42:28,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:28,221 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:28,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:28,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:28,254 INFO [Thread-1145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1145. 2024-11-07T12:42:28,254 DEBUG [Thread-1145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:42:28,254 DEBUG [Thread-1145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:42:28,254 DEBUG [Thread-1145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:42:28,254 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:42:28,261 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=598 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/80a435f2dec24d28b909f0a2f35181c0 2024-11-07T12:42:28,263 INFO [Thread-1137 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1137. 2024-11-07T12:42:28,263 DEBUG [Thread-1137 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:42:28,263 DEBUG [Thread-1137 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:42:28,263 DEBUG [Thread-1137 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:42:28,263 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:42:28,270 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/df87e32e87c94789bb528a79f972ccbc is 50, key is test_row_0/C:col10/1730983347391/Put/seqid=0 2024-11-07T12:42:28,273 INFO [Thread-1143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1143. 2024-11-07T12:42:28,273 DEBUG [Thread-1143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:42:28,273 DEBUG [Thread-1143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:42:28,274 DEBUG [Thread-1143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:42:28,274 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:42:28,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742203_1379 (size=12301) 2024-11-07T12:42:28,379 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-07T12:42:28,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:28,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. as already flushing 2024-11-07T12:42:28,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:28,380 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:28,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:28,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:28,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-07T12:42:28,533 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-07T12:42:28,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:28,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. as already flushing 2024-11-07T12:42:28,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:28,533 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:28,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:28,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:28,685 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=598 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/df87e32e87c94789bb528a79f972ccbc 2024-11-07T12:42:28,686 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-07T12:42:28,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:28,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. as already flushing 2024-11-07T12:42:28,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:28,687 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:28,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:28,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:28,691 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/8a8ab913fbda402abe9d3b5ec6a58994 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/8a8ab913fbda402abe9d3b5ec6a58994 2024-11-07T12:42:28,695 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/8a8ab913fbda402abe9d3b5ec6a58994, entries=150, sequenceid=598, filesize=12.0 K 2024-11-07T12:42:28,695 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/80a435f2dec24d28b909f0a2f35181c0 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/80a435f2dec24d28b909f0a2f35181c0 2024-11-07T12:42:28,699 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/80a435f2dec24d28b909f0a2f35181c0, entries=150, sequenceid=598, filesize=12.0 K 2024-11-07T12:42:28,700 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/df87e32e87c94789bb528a79f972ccbc as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/df87e32e87c94789bb528a79f972ccbc 2024-11-07T12:42:28,703 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/df87e32e87c94789bb528a79f972ccbc, entries=150, sequenceid=598, filesize=12.0 K 2024-11-07T12:42:28,704 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=20.13 KB/20610 for 8dc13886c19c8110f075e43282960321 in 1302ms, sequenceid=598, compaction requested=false 2024-11-07T12:42:28,704 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8dc13886c19c8110f075e43282960321: 2024-11-07T12:42:28,840 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-07T12:42:28,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:28,841 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2902): Flushing 8dc13886c19c8110f075e43282960321 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-07T12:42:28,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=A 2024-11-07T12:42:28,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:28,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=B 2024-11-07T12:42:28,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:28,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8dc13886c19c8110f075e43282960321, store=C 2024-11-07T12:42:28,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:28,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/05541d424f5a43268e1dbc422515b30e is 50, key is test_row_0/A:col10/1730983348272/Put/seqid=0 2024-11-07T12:42:28,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742204_1380 (size=9857) 2024-11-07T12:42:29,249 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=607 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/05541d424f5a43268e1dbc422515b30e 2024-11-07T12:42:29,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/0684e5ef8bd14f06bb8b8f57a246aec2 is 50, key is test_row_0/B:col10/1730983348272/Put/seqid=0 2024-11-07T12:42:29,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742205_1381 (size=9857) 2024-11-07T12:42:29,661 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=607 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/0684e5ef8bd14f06bb8b8f57a246aec2 2024-11-07T12:42:29,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/bd5734397c984659bf1d26c739ab0050 is 50, key is test_row_0/C:col10/1730983348272/Put/seqid=0 2024-11-07T12:42:29,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742206_1382 (size=9857) 2024-11-07T12:42:30,082 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=607 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/bd5734397c984659bf1d26c739ab0050 2024-11-07T12:42:30,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/A/05541d424f5a43268e1dbc422515b30e as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/05541d424f5a43268e1dbc422515b30e 2024-11-07T12:42:30,089 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/05541d424f5a43268e1dbc422515b30e, entries=100, sequenceid=607, filesize=9.6 K 2024-11-07T12:42:30,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/B/0684e5ef8bd14f06bb8b8f57a246aec2 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/0684e5ef8bd14f06bb8b8f57a246aec2 2024-11-07T12:42:30,092 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/0684e5ef8bd14f06bb8b8f57a246aec2, entries=100, sequenceid=607, filesize=9.6 K 2024-11-07T12:42:30,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/.tmp/C/bd5734397c984659bf1d26c739ab0050 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/bd5734397c984659bf1d26c739ab0050 2024-11-07T12:42:30,095 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/bd5734397c984659bf1d26c739ab0050, entries=100, sequenceid=607, filesize=9.6 K 2024-11-07T12:42:30,096 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 8dc13886c19c8110f075e43282960321 in 1256ms, sequenceid=607, compaction requested=true 2024-11-07T12:42:30,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2603): Flush status journal for 8dc13886c19c8110f075e43282960321: 2024-11-07T12:42:30,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:30,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=85 2024-11-07T12:42:30,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.HMaster(4169): Remote procedure done, pid=85 2024-11-07T12:42:30,099 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-07T12:42:30,099 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.7690 sec 2024-11-07T12:42:30,101 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees in 3.7750 sec 2024-11-07T12:42:30,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-07T12:42:30,492 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:42:30,492 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-07T12:42:30,492 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 66 2024-11-07T12:42:30,492 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 126 2024-11-07T12:42:30,492 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 117 2024-11-07T12:42:30,492 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-11-07T12:42:30,492 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 95 2024-11-07T12:42:30,492 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-07T12:42:30,492 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2181 2024-11-07T12:42:30,492 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2191 2024-11-07T12:42:30,492 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2134 2024-11-07T12:42:30,492 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2215 2024-11-07T12:42:30,492 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2212 2024-11-07T12:42:30,492 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-07T12:42:30,492 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-07T12:42:30,492 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T12:42:30,492 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:42:30,492 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:42:30,492 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:42:30,493 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-07T12:42:30,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-07T12:42:30,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-07T12:42:30,497 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730983350496"}]},"ts":"1730983350496"} 2024-11-07T12:42:30,498 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-07T12:42:30,498 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-07T12:42:30,499 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-07T12:42:30,500 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8dc13886c19c8110f075e43282960321, UNASSIGN}] 2024-11-07T12:42:30,501 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8dc13886c19c8110f075e43282960321, UNASSIGN 2024-11-07T12:42:30,501 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=8dc13886c19c8110f075e43282960321, regionState=CLOSING, regionLocation=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:30,503 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8dc13886c19c8110f075e43282960321, UNASSIGN because future has completed 2024-11-07T12:42:30,503 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-07T12:42:30,503 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217}] 2024-11-07T12:42:30,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-07T12:42:30,655 INFO [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(122): Close 8dc13886c19c8110f075e43282960321 2024-11-07T12:42:30,655 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-07T12:42:30,655 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1722): Closing 8dc13886c19c8110f075e43282960321, disabling compactions & flushes 2024-11-07T12:42:30,655 INFO [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:30,655 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:30,655 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. after waiting 0 ms 2024-11-07T12:42:30,655 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:30,656 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/04a388aed75a40b0ad01201ef25b1bab, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/e6041cbd6b634c7083d035ea8d8da73e, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/3b3c7ae72d1f4c90939fd94c1a32d8cc, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/240af32ce2724ad996f2a04c71cb3710, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/fd2e5fefe2a646e7a29a6913a01a422a, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/3c2bdc66274549b1b9220c4982e484ca, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/551c2dbdd1ee47e9850fe0ad64a35d55, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/0e12360be28f435da4ddefb2996a9077, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/f4c22dffcd63417cba781ba4a7661ef6, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/79dd3052429841f1b5380ea68d0a7be1, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/028e09b4fe5346239a93a80b44fb8fea, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/c65833e71e824b6b833ea4f62a125fbc, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/3a9f823b96a044fe8612cb11c76313b2, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/02be950e09bd4f30b7529f46f9dbb613, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/6a23d80af0084823afa4d4d2ef294d4b, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/557f0816c56c43158adc93f3dd02e23e, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/09d00082c9ae41a49edb4f1e9dfc399f, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/a3a9d59ba7d649cbaa6fb98c1626fdbc, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/9ed670d3d0a24dbb9cd78f7450bda788, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/5f0be8de0746483cad0da1ce0ccc4671, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/316944d61b8d4e2ab5c7e585b3e0ebae, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/3015daabefb54d8295ddc554ff3750b7, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/c7ff1c1a9a6143fcba89c5bc815231d6, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/bb7bd17bf41e47b78f8f8ef571c5f6ab, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/e0baccc0f1724da2bf253afb89193399, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/fe649630ed9b4bea9931215aca3fcabf, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/cfa3d4bc2d7c436eaa27f7c5b7fe98aa, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/83d995e5e424409ba9842c59786264d7, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/a0a1958157724b779615a421d569a2cf, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/932f933799264cdab336d4487c6b53b3, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/9395bc02a5734172815e7119d3ece447, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/43372c6fdbe2423b81daac3750155cf2, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/2e392408f4ca462ba870f1c15c92f4d0, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/0949ef2b947e4f63900de72628f3a73f, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/25e5e7eacdf04a62a55257ec1ab749ac, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/d67f5b4de2c44e3ba7fd66d65509e95b, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/417f7434e70d4e8f9f5180d203e328ea, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/747cdf3fd7aa4bce95ee27322da9fa6d, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/4f6f9d89aeae4be99adc7124f96bcc23, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/ae2cf5bebcec4934959cd90d334afd4f, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/95c60ea6737e411b942c0f990adff55c, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/21e737d8256c48968b0bb2535df3841f, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/bb308c9e0a14413cb3f2c684498a6c8e] to archive 2024-11-07T12:42:30,657 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T12:42:30,659 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/04a388aed75a40b0ad01201ef25b1bab to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/04a388aed75a40b0ad01201ef25b1bab 2024-11-07T12:42:30,661 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/e6041cbd6b634c7083d035ea8d8da73e to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/e6041cbd6b634c7083d035ea8d8da73e 2024-11-07T12:42:30,661 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/3b3c7ae72d1f4c90939fd94c1a32d8cc to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/3b3c7ae72d1f4c90939fd94c1a32d8cc 2024-11-07T12:42:30,662 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/240af32ce2724ad996f2a04c71cb3710 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/240af32ce2724ad996f2a04c71cb3710 2024-11-07T12:42:30,664 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/fd2e5fefe2a646e7a29a6913a01a422a to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/fd2e5fefe2a646e7a29a6913a01a422a 2024-11-07T12:42:30,665 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/3c2bdc66274549b1b9220c4982e484ca to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/3c2bdc66274549b1b9220c4982e484ca 2024-11-07T12:42:30,666 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/551c2dbdd1ee47e9850fe0ad64a35d55 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/551c2dbdd1ee47e9850fe0ad64a35d55 2024-11-07T12:42:30,666 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/0e12360be28f435da4ddefb2996a9077 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/0e12360be28f435da4ddefb2996a9077 2024-11-07T12:42:30,667 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/f4c22dffcd63417cba781ba4a7661ef6 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/f4c22dffcd63417cba781ba4a7661ef6 2024-11-07T12:42:30,668 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/79dd3052429841f1b5380ea68d0a7be1 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/79dd3052429841f1b5380ea68d0a7be1 2024-11-07T12:42:30,669 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/028e09b4fe5346239a93a80b44fb8fea to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/028e09b4fe5346239a93a80b44fb8fea 2024-11-07T12:42:30,670 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/c65833e71e824b6b833ea4f62a125fbc to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/c65833e71e824b6b833ea4f62a125fbc 2024-11-07T12:42:30,671 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/3a9f823b96a044fe8612cb11c76313b2 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/3a9f823b96a044fe8612cb11c76313b2 2024-11-07T12:42:30,672 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/02be950e09bd4f30b7529f46f9dbb613 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/02be950e09bd4f30b7529f46f9dbb613 2024-11-07T12:42:30,673 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/6a23d80af0084823afa4d4d2ef294d4b to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/6a23d80af0084823afa4d4d2ef294d4b 2024-11-07T12:42:30,674 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/557f0816c56c43158adc93f3dd02e23e to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/557f0816c56c43158adc93f3dd02e23e 2024-11-07T12:42:30,675 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/09d00082c9ae41a49edb4f1e9dfc399f to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/09d00082c9ae41a49edb4f1e9dfc399f 2024-11-07T12:42:30,676 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/a3a9d59ba7d649cbaa6fb98c1626fdbc to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/a3a9d59ba7d649cbaa6fb98c1626fdbc 2024-11-07T12:42:30,677 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/9ed670d3d0a24dbb9cd78f7450bda788 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/9ed670d3d0a24dbb9cd78f7450bda788 2024-11-07T12:42:30,678 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/5f0be8de0746483cad0da1ce0ccc4671 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/5f0be8de0746483cad0da1ce0ccc4671 2024-11-07T12:42:30,679 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/316944d61b8d4e2ab5c7e585b3e0ebae to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/316944d61b8d4e2ab5c7e585b3e0ebae 2024-11-07T12:42:30,680 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/3015daabefb54d8295ddc554ff3750b7 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/3015daabefb54d8295ddc554ff3750b7 2024-11-07T12:42:30,682 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/c7ff1c1a9a6143fcba89c5bc815231d6 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/c7ff1c1a9a6143fcba89c5bc815231d6 2024-11-07T12:42:30,683 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/bb7bd17bf41e47b78f8f8ef571c5f6ab to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/bb7bd17bf41e47b78f8f8ef571c5f6ab 2024-11-07T12:42:30,684 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/e0baccc0f1724da2bf253afb89193399 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/e0baccc0f1724da2bf253afb89193399 2024-11-07T12:42:30,685 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/fe649630ed9b4bea9931215aca3fcabf to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/fe649630ed9b4bea9931215aca3fcabf 2024-11-07T12:42:30,686 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/cfa3d4bc2d7c436eaa27f7c5b7fe98aa to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/cfa3d4bc2d7c436eaa27f7c5b7fe98aa 2024-11-07T12:42:30,687 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/83d995e5e424409ba9842c59786264d7 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/83d995e5e424409ba9842c59786264d7 2024-11-07T12:42:30,688 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/a0a1958157724b779615a421d569a2cf to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/a0a1958157724b779615a421d569a2cf 2024-11-07T12:42:30,688 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/932f933799264cdab336d4487c6b53b3 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/932f933799264cdab336d4487c6b53b3 2024-11-07T12:42:30,689 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/9395bc02a5734172815e7119d3ece447 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/9395bc02a5734172815e7119d3ece447 2024-11-07T12:42:30,690 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/43372c6fdbe2423b81daac3750155cf2 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/43372c6fdbe2423b81daac3750155cf2 2024-11-07T12:42:30,691 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/2e392408f4ca462ba870f1c15c92f4d0 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/2e392408f4ca462ba870f1c15c92f4d0 2024-11-07T12:42:30,692 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/0949ef2b947e4f63900de72628f3a73f to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/0949ef2b947e4f63900de72628f3a73f 2024-11-07T12:42:30,693 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/25e5e7eacdf04a62a55257ec1ab749ac to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/25e5e7eacdf04a62a55257ec1ab749ac 2024-11-07T12:42:30,694 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/d67f5b4de2c44e3ba7fd66d65509e95b to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/d67f5b4de2c44e3ba7fd66d65509e95b 2024-11-07T12:42:30,695 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/417f7434e70d4e8f9f5180d203e328ea to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/417f7434e70d4e8f9f5180d203e328ea 2024-11-07T12:42:30,696 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/747cdf3fd7aa4bce95ee27322da9fa6d to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/747cdf3fd7aa4bce95ee27322da9fa6d 2024-11-07T12:42:30,697 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/4f6f9d89aeae4be99adc7124f96bcc23 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/4f6f9d89aeae4be99adc7124f96bcc23 2024-11-07T12:42:30,698 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/ae2cf5bebcec4934959cd90d334afd4f to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/ae2cf5bebcec4934959cd90d334afd4f 2024-11-07T12:42:30,699 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/95c60ea6737e411b942c0f990adff55c to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/95c60ea6737e411b942c0f990adff55c 2024-11-07T12:42:30,700 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/21e737d8256c48968b0bb2535df3841f to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/21e737d8256c48968b0bb2535df3841f 2024-11-07T12:42:30,701 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/bb308c9e0a14413cb3f2c684498a6c8e to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/bb308c9e0a14413cb3f2c684498a6c8e 2024-11-07T12:42:30,703 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/5c968edfd75f491db5fedc7a63589a8e, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/254b0d38fd9a4fcf80b0a0e3fcc72b82, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/e8e2bcccd7a14b779db96f8bc532ef96, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/68e78311dd5540afb6c30605edae96be, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/945a9355e07a4ac8ab5c9126bfd06075, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/65ccef9dff8d482f973b665056dac44f, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/de80696a81c943188a3a32024a92b8b8, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/a0f88cc157694d179447a4d2aba24fc1, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/f1f9e3ca46574b6fba6bfd1e572040ea, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/097d432acecf42118cfeee6ae22347f1, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/b4ed6e537731444b8c90a0bf75f8773c, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/7ef02e120f4f48e8ac6be67260650987, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/33948893b9364b75a3a7b522f3e2a5d6, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/2a391260d5344a6399f6486b3f7f9fc1, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/e0d27685b06e49169f4ca7ce1878caa3, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/b326d395a8c54cb6a850ed756cb8a9ba, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/25451cf92b4344738e84fa89ea662041, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/03c2ea01c3d54a3c858ce41cabaffaf0, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/b6a059a0f7be47ada66f32acfe122fc5, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/2459f784c8cd48f9a009974b7dade643, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/29deb66790584f99b4a69d0c19e03572, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/613372ea89ec4e6a962c0c708a56f19d, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/1e5298171c6f4d17b79939bd0d1d3a4a, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/519c8d48c87b47ebb01bb2236fb344d5, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/932f336f56794b99bed0d41f0dc1284a, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/05325fa851b84f4aa9cb074a78b1e97d, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/911434becb874227b5a097efedcc1879, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/cf774d9d1b3a47dd99d9cdf187c714c6, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/4a415639b238442983553587eeec51ce, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/bc8f460ec43a4c06aa37fe8f7994d1c4, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/7a97b9dbc5414a6ca69cfbb0a758f6f2, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/0263d7c342ce48158ac803795745cc13, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/6f89040571cb4bda9aab6eb7e8d461d3, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/47d126917a7045d8843c3eb1f4b5071c, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/c4e1484e0ef34006bcb6c892ea7ebd0b, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/bf219f9fdef7487884ff792c793c831e, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/25ced5386c5548379c750fa1f2de7c49, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/d7a160ce7eb54f96a0f27d4723952b9c, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/a1ff26a2b56346938b518881f6bea3ac, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/3e3a08ca55a34fd4a1397964f64de38c, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/683dfcf3088042038ea16c62f1bb63ad, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/a86aa6d7422448028405cbd45db67ff6, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/520d19ca1b70429d815aea427a2c5789] to archive 2024-11-07T12:42:30,704 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T12:42:30,705 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/5c968edfd75f491db5fedc7a63589a8e to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/5c968edfd75f491db5fedc7a63589a8e 2024-11-07T12:42:30,706 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/254b0d38fd9a4fcf80b0a0e3fcc72b82 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/254b0d38fd9a4fcf80b0a0e3fcc72b82 2024-11-07T12:42:30,707 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/e8e2bcccd7a14b779db96f8bc532ef96 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/e8e2bcccd7a14b779db96f8bc532ef96 2024-11-07T12:42:30,707 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/68e78311dd5540afb6c30605edae96be to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/68e78311dd5540afb6c30605edae96be 2024-11-07T12:42:30,708 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/945a9355e07a4ac8ab5c9126bfd06075 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/945a9355e07a4ac8ab5c9126bfd06075 2024-11-07T12:42:30,709 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/65ccef9dff8d482f973b665056dac44f to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/65ccef9dff8d482f973b665056dac44f 2024-11-07T12:42:30,710 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/de80696a81c943188a3a32024a92b8b8 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/de80696a81c943188a3a32024a92b8b8 2024-11-07T12:42:30,711 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/a0f88cc157694d179447a4d2aba24fc1 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/a0f88cc157694d179447a4d2aba24fc1 2024-11-07T12:42:30,712 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/f1f9e3ca46574b6fba6bfd1e572040ea to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/f1f9e3ca46574b6fba6bfd1e572040ea 2024-11-07T12:42:30,713 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/097d432acecf42118cfeee6ae22347f1 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/097d432acecf42118cfeee6ae22347f1 2024-11-07T12:42:30,714 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/b4ed6e537731444b8c90a0bf75f8773c to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/b4ed6e537731444b8c90a0bf75f8773c 2024-11-07T12:42:30,714 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/7ef02e120f4f48e8ac6be67260650987 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/7ef02e120f4f48e8ac6be67260650987 2024-11-07T12:42:30,715 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/33948893b9364b75a3a7b522f3e2a5d6 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/33948893b9364b75a3a7b522f3e2a5d6 2024-11-07T12:42:30,716 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/2a391260d5344a6399f6486b3f7f9fc1 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/2a391260d5344a6399f6486b3f7f9fc1 2024-11-07T12:42:30,717 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/e0d27685b06e49169f4ca7ce1878caa3 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/e0d27685b06e49169f4ca7ce1878caa3 2024-11-07T12:42:30,718 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/b326d395a8c54cb6a850ed756cb8a9ba to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/b326d395a8c54cb6a850ed756cb8a9ba 2024-11-07T12:42:30,719 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/25451cf92b4344738e84fa89ea662041 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/25451cf92b4344738e84fa89ea662041 2024-11-07T12:42:30,720 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/03c2ea01c3d54a3c858ce41cabaffaf0 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/03c2ea01c3d54a3c858ce41cabaffaf0 2024-11-07T12:42:30,721 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/b6a059a0f7be47ada66f32acfe122fc5 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/b6a059a0f7be47ada66f32acfe122fc5 2024-11-07T12:42:30,722 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/2459f784c8cd48f9a009974b7dade643 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/2459f784c8cd48f9a009974b7dade643 2024-11-07T12:42:30,723 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/29deb66790584f99b4a69d0c19e03572 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/29deb66790584f99b4a69d0c19e03572 2024-11-07T12:42:30,724 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/613372ea89ec4e6a962c0c708a56f19d to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/613372ea89ec4e6a962c0c708a56f19d 2024-11-07T12:42:30,725 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/1e5298171c6f4d17b79939bd0d1d3a4a to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/1e5298171c6f4d17b79939bd0d1d3a4a 2024-11-07T12:42:30,726 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/519c8d48c87b47ebb01bb2236fb344d5 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/519c8d48c87b47ebb01bb2236fb344d5 2024-11-07T12:42:30,727 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/932f336f56794b99bed0d41f0dc1284a to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/932f336f56794b99bed0d41f0dc1284a 2024-11-07T12:42:30,728 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/05325fa851b84f4aa9cb074a78b1e97d to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/05325fa851b84f4aa9cb074a78b1e97d 2024-11-07T12:42:30,729 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/911434becb874227b5a097efedcc1879 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/911434becb874227b5a097efedcc1879 2024-11-07T12:42:30,730 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/cf774d9d1b3a47dd99d9cdf187c714c6 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/cf774d9d1b3a47dd99d9cdf187c714c6 2024-11-07T12:42:30,730 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/4a415639b238442983553587eeec51ce to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/4a415639b238442983553587eeec51ce 2024-11-07T12:42:30,731 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/bc8f460ec43a4c06aa37fe8f7994d1c4 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/bc8f460ec43a4c06aa37fe8f7994d1c4 2024-11-07T12:42:30,732 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/7a97b9dbc5414a6ca69cfbb0a758f6f2 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/7a97b9dbc5414a6ca69cfbb0a758f6f2 2024-11-07T12:42:30,733 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/0263d7c342ce48158ac803795745cc13 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/0263d7c342ce48158ac803795745cc13 2024-11-07T12:42:30,734 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/6f89040571cb4bda9aab6eb7e8d461d3 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/6f89040571cb4bda9aab6eb7e8d461d3 2024-11-07T12:42:30,735 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/47d126917a7045d8843c3eb1f4b5071c to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/47d126917a7045d8843c3eb1f4b5071c 2024-11-07T12:42:30,736 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/c4e1484e0ef34006bcb6c892ea7ebd0b to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/c4e1484e0ef34006bcb6c892ea7ebd0b 2024-11-07T12:42:30,737 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/bf219f9fdef7487884ff792c793c831e to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/bf219f9fdef7487884ff792c793c831e 2024-11-07T12:42:30,738 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/25ced5386c5548379c750fa1f2de7c49 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/25ced5386c5548379c750fa1f2de7c49 2024-11-07T12:42:30,738 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/d7a160ce7eb54f96a0f27d4723952b9c to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/d7a160ce7eb54f96a0f27d4723952b9c 2024-11-07T12:42:30,739 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/a1ff26a2b56346938b518881f6bea3ac to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/a1ff26a2b56346938b518881f6bea3ac 2024-11-07T12:42:30,740 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/3e3a08ca55a34fd4a1397964f64de38c to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/3e3a08ca55a34fd4a1397964f64de38c 2024-11-07T12:42:30,741 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/683dfcf3088042038ea16c62f1bb63ad to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/683dfcf3088042038ea16c62f1bb63ad 2024-11-07T12:42:30,742 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/a86aa6d7422448028405cbd45db67ff6 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/a86aa6d7422448028405cbd45db67ff6 2024-11-07T12:42:30,743 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/520d19ca1b70429d815aea427a2c5789 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/520d19ca1b70429d815aea427a2c5789 2024-11-07T12:42:30,744 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/07f8545df3d647dfb97011bf2da37603, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/883d4efb87a34329bc3324daa3403b1a, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/9faf3b34597640e38adc0ec9855c08a0, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/06c86adf7f6342a2b61c285712d237d0, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/eb67a0705e3048beb64e6920ab33f7aa, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/7ea3b631081a49a28ef38de44c3aac48, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/6d94293b6e0c4b9a8d688d746be6a135, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/2da541cdae284a21bf92afb554eab79b, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/0d2ec7477807410385f2f33995e21377, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/4d4c03d2659d46d4b0c5e71684b866b0, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/d2b18571482a46be9d0cb6c95864a1ec, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/6a5809c7514e4b2a9a47a631fca8a662, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/7ca0ade2a2f4482682d7474b13fb86a2, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/a43576dd9ea44aa5bc9de424cfa96e23, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/6a343ced76af4ca6aaa9cbd4b60c28d2, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/20dd8b947f5a40c8932bbdfb56e66849, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/737af3c578744549ad552f898f250fcf, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/e215b50ea1b343078b8865f6a7207db8, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/6e3e1f2415dc461aaad8b7cc306f2c82, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/b0a7cc5219394fa48f248e0a9ac8fd44, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/2882c2e15b4a4c1ca8acf97018af6a03, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/5b12a32838434a30bccd3af318159dd0, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/cf5597b105b042aa97fad28d3c8dfa8e, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/b71dd55b27304789aed52531e7846d9a, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/7d327d8bed53415caa6090ef44e67fa2, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/54e8bf43509946a0bdc24aced9b9d8b7, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/f3179830cc594a2596bd5f223cad1b2e, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/b166f92db6924e0c8737c4ad6df85437, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/2527a28eabfe429d9f9a7c6ef917bcb2, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/6ede7d2abb66452e84ddbc6332ae69e1, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/90bbd66496184bc2be09eee5a79a5efe, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/d7e039431fee4462bb3c1a01ca53a0ff, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/29484e2d8a874546958567a53a29b7ec, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/25a8f16d8b6a4623be5fd581539121db, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/04470e32f61a4f56b54a9c7e5b655ac8, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/75038c1441f04469921acf3a4ac1da42, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/fcf19faee5644bf19ae27396b5cb7bde, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/a90cc24886e94f02a1a2d95e7abfbd87, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/5a441990bc0443658fdfb7d47a77f160, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/a536f9cc65894897be4709b941497bf4, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/66c7c195b2ed4884a0ff3a003f9a1508, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/3027b533ce53416d9a813df6bba1e9c4, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/19e1713981fd4d5da60d3b5cb6f983dc] to archive 2024-11-07T12:42:30,745 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T12:42:30,746 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/07f8545df3d647dfb97011bf2da37603 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/07f8545df3d647dfb97011bf2da37603 2024-11-07T12:42:30,747 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/883d4efb87a34329bc3324daa3403b1a to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/883d4efb87a34329bc3324daa3403b1a 2024-11-07T12:42:30,748 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/9faf3b34597640e38adc0ec9855c08a0 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/9faf3b34597640e38adc0ec9855c08a0 2024-11-07T12:42:30,749 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/06c86adf7f6342a2b61c285712d237d0 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/06c86adf7f6342a2b61c285712d237d0 2024-11-07T12:42:30,750 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/eb67a0705e3048beb64e6920ab33f7aa to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/eb67a0705e3048beb64e6920ab33f7aa 2024-11-07T12:42:30,751 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/7ea3b631081a49a28ef38de44c3aac48 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/7ea3b631081a49a28ef38de44c3aac48 2024-11-07T12:42:30,751 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/6d94293b6e0c4b9a8d688d746be6a135 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/6d94293b6e0c4b9a8d688d746be6a135 2024-11-07T12:42:30,752 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/2da541cdae284a21bf92afb554eab79b to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/2da541cdae284a21bf92afb554eab79b 2024-11-07T12:42:30,753 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/0d2ec7477807410385f2f33995e21377 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/0d2ec7477807410385f2f33995e21377 2024-11-07T12:42:30,754 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/4d4c03d2659d46d4b0c5e71684b866b0 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/4d4c03d2659d46d4b0c5e71684b866b0 2024-11-07T12:42:30,754 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/d2b18571482a46be9d0cb6c95864a1ec to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/d2b18571482a46be9d0cb6c95864a1ec 2024-11-07T12:42:30,755 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/6a5809c7514e4b2a9a47a631fca8a662 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/6a5809c7514e4b2a9a47a631fca8a662 2024-11-07T12:42:30,756 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/7ca0ade2a2f4482682d7474b13fb86a2 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/7ca0ade2a2f4482682d7474b13fb86a2 2024-11-07T12:42:30,757 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/a43576dd9ea44aa5bc9de424cfa96e23 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/a43576dd9ea44aa5bc9de424cfa96e23 2024-11-07T12:42:30,758 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/6a343ced76af4ca6aaa9cbd4b60c28d2 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/6a343ced76af4ca6aaa9cbd4b60c28d2 2024-11-07T12:42:30,759 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/20dd8b947f5a40c8932bbdfb56e66849 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/20dd8b947f5a40c8932bbdfb56e66849 2024-11-07T12:42:30,760 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/737af3c578744549ad552f898f250fcf to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/737af3c578744549ad552f898f250fcf 2024-11-07T12:42:30,760 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/e215b50ea1b343078b8865f6a7207db8 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/e215b50ea1b343078b8865f6a7207db8 2024-11-07T12:42:30,761 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/6e3e1f2415dc461aaad8b7cc306f2c82 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/6e3e1f2415dc461aaad8b7cc306f2c82 2024-11-07T12:42:30,762 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/b0a7cc5219394fa48f248e0a9ac8fd44 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/b0a7cc5219394fa48f248e0a9ac8fd44 2024-11-07T12:42:30,763 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/2882c2e15b4a4c1ca8acf97018af6a03 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/2882c2e15b4a4c1ca8acf97018af6a03 2024-11-07T12:42:30,764 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/5b12a32838434a30bccd3af318159dd0 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/5b12a32838434a30bccd3af318159dd0 2024-11-07T12:42:30,765 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/cf5597b105b042aa97fad28d3c8dfa8e to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/cf5597b105b042aa97fad28d3c8dfa8e 2024-11-07T12:42:30,765 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/b71dd55b27304789aed52531e7846d9a to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/b71dd55b27304789aed52531e7846d9a 2024-11-07T12:42:30,766 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/7d327d8bed53415caa6090ef44e67fa2 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/7d327d8bed53415caa6090ef44e67fa2 2024-11-07T12:42:30,767 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/54e8bf43509946a0bdc24aced9b9d8b7 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/54e8bf43509946a0bdc24aced9b9d8b7 2024-11-07T12:42:30,768 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/f3179830cc594a2596bd5f223cad1b2e to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/f3179830cc594a2596bd5f223cad1b2e 2024-11-07T12:42:30,769 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/b166f92db6924e0c8737c4ad6df85437 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/b166f92db6924e0c8737c4ad6df85437 2024-11-07T12:42:30,769 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/2527a28eabfe429d9f9a7c6ef917bcb2 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/2527a28eabfe429d9f9a7c6ef917bcb2 2024-11-07T12:42:30,770 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/6ede7d2abb66452e84ddbc6332ae69e1 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/6ede7d2abb66452e84ddbc6332ae69e1 2024-11-07T12:42:30,771 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/90bbd66496184bc2be09eee5a79a5efe to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/90bbd66496184bc2be09eee5a79a5efe 2024-11-07T12:42:30,772 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/d7e039431fee4462bb3c1a01ca53a0ff to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/d7e039431fee4462bb3c1a01ca53a0ff 2024-11-07T12:42:30,773 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/29484e2d8a874546958567a53a29b7ec to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/29484e2d8a874546958567a53a29b7ec 2024-11-07T12:42:30,773 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/25a8f16d8b6a4623be5fd581539121db to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/25a8f16d8b6a4623be5fd581539121db 2024-11-07T12:42:30,774 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/04470e32f61a4f56b54a9c7e5b655ac8 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/04470e32f61a4f56b54a9c7e5b655ac8 2024-11-07T12:42:30,775 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/75038c1441f04469921acf3a4ac1da42 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/75038c1441f04469921acf3a4ac1da42 2024-11-07T12:42:30,776 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/fcf19faee5644bf19ae27396b5cb7bde to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/fcf19faee5644bf19ae27396b5cb7bde 2024-11-07T12:42:30,777 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/a90cc24886e94f02a1a2d95e7abfbd87 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/a90cc24886e94f02a1a2d95e7abfbd87 2024-11-07T12:42:30,778 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/5a441990bc0443658fdfb7d47a77f160 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/5a441990bc0443658fdfb7d47a77f160 2024-11-07T12:42:30,778 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/a536f9cc65894897be4709b941497bf4 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/a536f9cc65894897be4709b941497bf4 2024-11-07T12:42:30,780 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/66c7c195b2ed4884a0ff3a003f9a1508 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/66c7c195b2ed4884a0ff3a003f9a1508 2024-11-07T12:42:30,781 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/3027b533ce53416d9a813df6bba1e9c4 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/3027b533ce53416d9a813df6bba1e9c4 2024-11-07T12:42:30,781 DEBUG [StoreCloser-TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/19e1713981fd4d5da60d3b5cb6f983dc to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/19e1713981fd4d5da60d3b5cb6f983dc 2024-11-07T12:42:30,795 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/recovered.edits/610.seqid, newMaxSeqId=610, maxSeqId=1 2024-11-07T12:42:30,796 INFO [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321. 2024-11-07T12:42:30,796 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1676): Region close journal for 8dc13886c19c8110f075e43282960321: Waiting for close lock at 1730983350655Running coprocessor pre-close hooks at 1730983350655Disabling compacts and flushes for region at 1730983350655Disabling writes for close at 1730983350655Writing region close event to WAL at 1730983350783 (+128 ms)Running coprocessor post-close hooks at 1730983350795 (+12 ms)Closed at 1730983350796 (+1 ms) 2024-11-07T12:42:30,798 INFO [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(157): Closed 8dc13886c19c8110f075e43282960321 2024-11-07T12:42:30,798 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=8dc13886c19c8110f075e43282960321, regionState=CLOSED 2024-11-07T12:42:30,800 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 because future has completed 2024-11-07T12:42:30,802 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-07T12:42:30,802 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; CloseRegionProcedure 8dc13886c19c8110f075e43282960321, server=db9ad1cb6cf9,34293,1730983260217 in 297 msec 2024-11-07T12:42:30,803 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-11-07T12:42:30,803 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8dc13886c19c8110f075e43282960321, UNASSIGN in 302 msec 2024-11-07T12:42:30,805 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-07T12:42:30,805 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 305 msec 2024-11-07T12:42:30,807 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730983350806"}]},"ts":"1730983350806"} 2024-11-07T12:42:30,808 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-07T12:42:30,808 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-07T12:42:30,809 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 315 msec 2024-11-07T12:42:30,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-07T12:42:30,812 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:42:30,813 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-07T12:42:30,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:42:30,814 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=90, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:42:30,814 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=90, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:42:30,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-07T12:42:30,816 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321 2024-11-07T12:42:30,818 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A, FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B, FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C, FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/recovered.edits] 2024-11-07T12:42:30,820 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/05541d424f5a43268e1dbc422515b30e to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/05541d424f5a43268e1dbc422515b30e 2024-11-07T12:42:30,822 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/8a8ab913fbda402abe9d3b5ec6a58994 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/8a8ab913fbda402abe9d3b5ec6a58994 2024-11-07T12:42:30,823 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/943794d6132d46bb8d9a7828dc9a431b to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/A/943794d6132d46bb8d9a7828dc9a431b 2024-11-07T12:42:30,825 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/0684e5ef8bd14f06bb8b8f57a246aec2 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/0684e5ef8bd14f06bb8b8f57a246aec2 2024-11-07T12:42:30,826 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/80a435f2dec24d28b909f0a2f35181c0 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/80a435f2dec24d28b909f0a2f35181c0 2024-11-07T12:42:30,827 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/9a21fdc5f989431a8503b0895c21dd43 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/B/9a21fdc5f989431a8503b0895c21dd43 2024-11-07T12:42:30,828 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/0ca07d34d37049bd8e51df149abea9bd to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/0ca07d34d37049bd8e51df149abea9bd 2024-11-07T12:42:30,829 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/bd5734397c984659bf1d26c739ab0050 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/bd5734397c984659bf1d26c739ab0050 2024-11-07T12:42:30,830 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/df87e32e87c94789bb528a79f972ccbc to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/C/df87e32e87c94789bb528a79f972ccbc 2024-11-07T12:42:30,833 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/recovered.edits/610.seqid to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321/recovered.edits/610.seqid 2024-11-07T12:42:30,834 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/8dc13886c19c8110f075e43282960321 2024-11-07T12:42:30,834 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-07T12:42:30,836 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=90, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:42:30,839 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-07T12:42:30,841 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-07T12:42:30,842 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=90, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:42:30,842 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-07T12:42:30,843 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1730983350842"}]},"ts":"9223372036854775807"} 2024-11-07T12:42:30,845 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-07T12:42:30,845 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 8dc13886c19c8110f075e43282960321, NAME => 'TestAcidGuarantees,,1730983326572.8dc13886c19c8110f075e43282960321.', STARTKEY => '', ENDKEY => ''}] 2024-11-07T12:42:30,845 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-07T12:42:30,845 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1730983350845"}]},"ts":"9223372036854775807"} 2024-11-07T12:42:30,847 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-07T12:42:30,848 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=90, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:42:30,849 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 35 msec 2024-11-07T12:42:30,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-07T12:42:30,922 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-07T12:42:30,923 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:42:30,937 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=252 (was 251) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1483776571_22 at /127.0.0.1:34356 [Waiting for operation #1225] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1483776571_22 at /127.0.0.1:34416 [Waiting for operation #1226] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-269572622_22 at /127.0.0.1:34434 [Waiting for operation #938] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-269572622_22 at /127.0.0.1:42604 [Waiting for operation #177] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-269572622_22 at /127.0.0.1:54914 [Waiting for operation #1246] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=465 (was 463) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1073 (was 1224), ProcessCount=11 (was 11), AvailableMemoryMB=3080 (was 2622) - AvailableMemoryMB LEAK? - 2024-11-07T12:42:30,949 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=252, OpenFileDescriptor=465, MaxFileDescriptor=1048576, SystemLoadAverage=1073, ProcessCount=11, AvailableMemoryMB=3080 2024-11-07T12:42:30,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-07T12:42:30,951 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T12:42:30,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-07T12:42:30,953 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T12:42:30,953 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:30,953 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 91 2024-11-07T12:42:30,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-07T12:42:30,954 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T12:42:30,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742207_1383 (size=1061) 2024-11-07T12:42:31,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-07T12:42:31,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-07T12:42:31,364 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4 2024-11-07T12:42:31,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742208_1384 (size=53) 2024-11-07T12:42:31,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-07T12:42:31,771 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:42:31,771 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing ffbbe32019fd517ea21eab76e142aa7f, disabling compactions & flushes 2024-11-07T12:42:31,771 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:31,771 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:31,771 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. after waiting 0 ms 2024-11-07T12:42:31,771 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:31,771 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:31,771 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for ffbbe32019fd517ea21eab76e142aa7f: Waiting for close lock at 1730983351771Disabling compacts and flushes for region at 1730983351771Disabling writes for close at 1730983351771Writing region close event to WAL at 1730983351771Closed at 1730983351771 2024-11-07T12:42:31,772 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T12:42:31,772 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1730983351772"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730983351772"}]},"ts":"1730983351772"} 2024-11-07T12:42:31,774 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-07T12:42:31,775 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T12:42:31,775 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730983351775"}]},"ts":"1730983351775"} 2024-11-07T12:42:31,777 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-07T12:42:31,777 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ffbbe32019fd517ea21eab76e142aa7f, ASSIGN}] 2024-11-07T12:42:31,778 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ffbbe32019fd517ea21eab76e142aa7f, ASSIGN 2024-11-07T12:42:31,779 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ffbbe32019fd517ea21eab76e142aa7f, ASSIGN; state=OFFLINE, location=db9ad1cb6cf9,34293,1730983260217; forceNewPlan=false, retain=false 2024-11-07T12:42:31,930 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=ffbbe32019fd517ea21eab76e142aa7f, regionState=OPENING, regionLocation=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:31,932 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ffbbe32019fd517ea21eab76e142aa7f, ASSIGN because future has completed 2024-11-07T12:42:31,932 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217}] 2024-11-07T12:42:32,086 INFO [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:32,086 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(7752): Opening region: {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} 2024-11-07T12:42:32,086 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:32,086 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:42:32,086 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(7794): checking encryption for ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:32,086 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(7797): checking classloading for ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:32,088 INFO [StoreOpener-ffbbe32019fd517ea21eab76e142aa7f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:32,090 INFO [StoreOpener-ffbbe32019fd517ea21eab76e142aa7f-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-07T12:42:32,090 INFO [StoreOpener-ffbbe32019fd517ea21eab76e142aa7f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ffbbe32019fd517ea21eab76e142aa7f columnFamilyName A 2024-11-07T12:42:32,090 DEBUG [StoreOpener-ffbbe32019fd517ea21eab76e142aa7f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:32,090 INFO [StoreOpener-ffbbe32019fd517ea21eab76e142aa7f-1 {}] regionserver.HStore(327): Store=ffbbe32019fd517ea21eab76e142aa7f/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:42:32,090 INFO [StoreOpener-ffbbe32019fd517ea21eab76e142aa7f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:32,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-07T12:42:32,091 INFO [StoreOpener-ffbbe32019fd517ea21eab76e142aa7f-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-07T12:42:32,092 INFO [StoreOpener-ffbbe32019fd517ea21eab76e142aa7f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ffbbe32019fd517ea21eab76e142aa7f columnFamilyName B 2024-11-07T12:42:32,092 DEBUG [StoreOpener-ffbbe32019fd517ea21eab76e142aa7f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:32,092 INFO [StoreOpener-ffbbe32019fd517ea21eab76e142aa7f-1 {}] regionserver.HStore(327): Store=ffbbe32019fd517ea21eab76e142aa7f/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:42:32,092 INFO [StoreOpener-ffbbe32019fd517ea21eab76e142aa7f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:32,093 INFO [StoreOpener-ffbbe32019fd517ea21eab76e142aa7f-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-07T12:42:32,093 INFO [StoreOpener-ffbbe32019fd517ea21eab76e142aa7f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ffbbe32019fd517ea21eab76e142aa7f columnFamilyName C 2024-11-07T12:42:32,093 DEBUG [StoreOpener-ffbbe32019fd517ea21eab76e142aa7f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:32,093 INFO [StoreOpener-ffbbe32019fd517ea21eab76e142aa7f-1 {}] regionserver.HStore(327): Store=ffbbe32019fd517ea21eab76e142aa7f/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:42:32,094 INFO [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:32,094 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1038): replaying wal for ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:32,094 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:32,095 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:32,095 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1048): stopping wal replay for ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:32,095 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1060): Cleaning up temporary data for ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:32,096 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-07T12:42:32,097 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1093): writing seq id for ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:32,099 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T12:42:32,099 INFO [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1114): Opened ffbbe32019fd517ea21eab76e142aa7f; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72644751, jitterRate=0.08249114453792572}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-07T12:42:32,099 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:32,100 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1006): Region open journal for ffbbe32019fd517ea21eab76e142aa7f: Running coprocessor pre-open hook at 1730983352087Writing region info on filesystem at 1730983352087Initializing all the Stores at 1730983352087Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730983352088 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730983352088Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730983352088Cleaning up temporary data from old regions at 1730983352095 (+7 ms)Running coprocessor post-open hooks at 1730983352099 (+4 ms)Region opened successfully at 1730983352099 2024-11-07T12:42:32,100 INFO [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., pid=93, masterSystemTime=1730983352083 2024-11-07T12:42:32,102 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:32,102 INFO [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:32,103 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=ffbbe32019fd517ea21eab76e142aa7f, regionState=OPEN, openSeqNum=2, regionLocation=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:32,104 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=93, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 because future has completed 2024-11-07T12:42:32,107 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-07T12:42:32,107 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; OpenRegionProcedure ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 in 173 msec 2024-11-07T12:42:32,109 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-07T12:42:32,109 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ffbbe32019fd517ea21eab76e142aa7f, ASSIGN in 330 msec 2024-11-07T12:42:32,110 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T12:42:32,110 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730983352110"}]},"ts":"1730983352110"} 2024-11-07T12:42:32,111 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-07T12:42:32,112 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T12:42:32,113 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1610 sec 2024-11-07T12:42:33,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-07T12:42:33,102 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:42:33,103 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@197fefe0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:42:33,103 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request db9ad1cb6cf9,38619,-1 for getting cluster id 2024-11-07T12:42:33,104 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:42:33,105 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e542096-4b0e-4b01-9952-662501fc0f65' 2024-11-07T12:42:33,105 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:42:33,105 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e542096-4b0e-4b01-9952-662501fc0f65" 2024-11-07T12:42:33,105 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39bc69b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:42:33,105 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [db9ad1cb6cf9,38619,-1] 2024-11-07T12:42:33,106 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:42:33,106 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:42:33,107 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50704, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:42:33,107 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@972c559, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:42:33,108 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:42:33,108 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=-1] 2024-11-07T12:42:33,108 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:42:33,109 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55788, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:42:33,110 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-07T12:42:33,111 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is db9ad1cb6cf9,38619,1730983259119 2024-11-07T12:42:33,111 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@43edcdb0 2024-11-07T12:42:33,111 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-07T12:42:33,112 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50708, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-07T12:42:33,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-07T12:42:33,113 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T12:42:33,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] procedure2.ProcedureExecutor(1139): Stored pid=94, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-07T12:42:33,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742209_1385 (size=1097) 2024-11-07T12:42:33,526 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-07T12:42:33,526 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-07T12:42:33,530 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-07T12:42:33,534 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ffbbe32019fd517ea21eab76e142aa7f, REOPEN/MOVE}] 2024-11-07T12:42:33,535 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ffbbe32019fd517ea21eab76e142aa7f, REOPEN/MOVE 2024-11-07T12:42:33,536 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=ffbbe32019fd517ea21eab76e142aa7f, regionState=CLOSING, regionLocation=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:33,538 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ffbbe32019fd517ea21eab76e142aa7f, REOPEN/MOVE because future has completed 2024-11-07T12:42:33,538 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-07T12:42:33,538 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=97, ppid=96, state=RUNNABLE, hasLock=false; CloseRegionProcedure ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217}] 2024-11-07T12:42:33,691 INFO [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(122): Close ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:33,691 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-07T12:42:33,691 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1722): Closing ffbbe32019fd517ea21eab76e142aa7f, disabling compactions & flushes 2024-11-07T12:42:33,691 INFO [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:33,691 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:33,691 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. after waiting 0 ms 2024-11-07T12:42:33,691 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:33,695 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-07T12:42:33,695 INFO [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:33,696 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1676): Region close journal for ffbbe32019fd517ea21eab76e142aa7f: Waiting for close lock at 1730983353691Running coprocessor pre-close hooks at 1730983353691Disabling compacts and flushes for region at 1730983353691Disabling writes for close at 1730983353691Writing region close event to WAL at 1730983353692 (+1 ms)Running coprocessor post-close hooks at 1730983353695 (+3 ms)Closed at 1730983353695 2024-11-07T12:42:33,696 WARN [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegionServer(3299): Not adding moved region record: ffbbe32019fd517ea21eab76e142aa7f to self. 2024-11-07T12:42:33,698 INFO [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(157): Closed ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:33,698 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=ffbbe32019fd517ea21eab76e142aa7f, regionState=CLOSED 2024-11-07T12:42:33,699 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=97, ppid=96, state=RUNNABLE, hasLock=false; CloseRegionProcedure ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 because future has completed 2024-11-07T12:42:33,701 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=97, resume processing ppid=96 2024-11-07T12:42:33,701 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, ppid=96, state=SUCCESS, hasLock=false; CloseRegionProcedure ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 in 162 msec 2024-11-07T12:42:33,702 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ffbbe32019fd517ea21eab76e142aa7f, REOPEN/MOVE; state=CLOSED, location=db9ad1cb6cf9,34293,1730983260217; forceNewPlan=false, retain=true 2024-11-07T12:42:33,853 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=ffbbe32019fd517ea21eab76e142aa7f, regionState=OPENING, regionLocation=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:33,854 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ffbbe32019fd517ea21eab76e142aa7f, REOPEN/MOVE because future has completed 2024-11-07T12:42:33,855 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=96, state=RUNNABLE, hasLock=false; OpenRegionProcedure ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217}] 2024-11-07T12:42:34,009 INFO [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:34,010 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(7752): Opening region: {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} 2024-11-07T12:42:34,010 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:34,010 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:42:34,010 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(7794): checking encryption for ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:34,010 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(7797): checking classloading for ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:34,011 INFO [StoreOpener-ffbbe32019fd517ea21eab76e142aa7f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:34,012 INFO [StoreOpener-ffbbe32019fd517ea21eab76e142aa7f-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-07T12:42:34,012 INFO [StoreOpener-ffbbe32019fd517ea21eab76e142aa7f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ffbbe32019fd517ea21eab76e142aa7f columnFamilyName A 2024-11-07T12:42:34,013 DEBUG [StoreOpener-ffbbe32019fd517ea21eab76e142aa7f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:34,014 INFO [StoreOpener-ffbbe32019fd517ea21eab76e142aa7f-1 {}] regionserver.HStore(327): Store=ffbbe32019fd517ea21eab76e142aa7f/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:42:34,014 INFO [StoreOpener-ffbbe32019fd517ea21eab76e142aa7f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:34,014 INFO [StoreOpener-ffbbe32019fd517ea21eab76e142aa7f-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-07T12:42:34,015 INFO [StoreOpener-ffbbe32019fd517ea21eab76e142aa7f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ffbbe32019fd517ea21eab76e142aa7f columnFamilyName B 2024-11-07T12:42:34,015 DEBUG [StoreOpener-ffbbe32019fd517ea21eab76e142aa7f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:34,015 INFO [StoreOpener-ffbbe32019fd517ea21eab76e142aa7f-1 {}] regionserver.HStore(327): Store=ffbbe32019fd517ea21eab76e142aa7f/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:42:34,015 INFO [StoreOpener-ffbbe32019fd517ea21eab76e142aa7f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:34,016 INFO [StoreOpener-ffbbe32019fd517ea21eab76e142aa7f-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-07T12:42:34,016 INFO [StoreOpener-ffbbe32019fd517ea21eab76e142aa7f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ffbbe32019fd517ea21eab76e142aa7f columnFamilyName C 2024-11-07T12:42:34,016 DEBUG [StoreOpener-ffbbe32019fd517ea21eab76e142aa7f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:34,016 INFO [StoreOpener-ffbbe32019fd517ea21eab76e142aa7f-1 {}] regionserver.HStore(327): Store=ffbbe32019fd517ea21eab76e142aa7f/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:42:34,016 INFO [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:34,016 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1038): replaying wal for ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:34,017 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:34,017 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:34,018 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1048): stopping wal replay for ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:34,018 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1060): Cleaning up temporary data for ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:34,019 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-07T12:42:34,020 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1093): writing seq id for ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:34,020 INFO [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1114): Opened ffbbe32019fd517ea21eab76e142aa7f; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74581761, jitterRate=0.11135484278202057}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-07T12:42:34,020 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:34,021 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1006): Region open journal for ffbbe32019fd517ea21eab76e142aa7f: Running coprocessor pre-open hook at 1730983354010Writing region info on filesystem at 1730983354010Initializing all the Stores at 1730983354011 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730983354011Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730983354011Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730983354011Cleaning up temporary data from old regions at 1730983354018 (+7 ms)Running coprocessor post-open hooks at 1730983354020 (+2 ms)Region opened successfully at 1730983354021 (+1 ms) 2024-11-07T12:42:34,022 INFO [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., pid=98, masterSystemTime=1730983354006 2024-11-07T12:42:34,023 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:34,024 INFO [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:34,024 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=ffbbe32019fd517ea21eab76e142aa7f, regionState=OPEN, openSeqNum=5, regionLocation=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:34,026 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=98, ppid=96, state=RUNNABLE, hasLock=false; OpenRegionProcedure ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 because future has completed 2024-11-07T12:42:34,028 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=96 2024-11-07T12:42:34,028 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=96, state=SUCCESS, hasLock=false; OpenRegionProcedure ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 in 172 msec 2024-11-07T12:42:34,029 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-07T12:42:34,030 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ffbbe32019fd517ea21eab76e142aa7f, REOPEN/MOVE in 494 msec 2024-11-07T12:42:34,031 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-11-07T12:42:34,032 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 500 msec 2024-11-07T12:42:34,033 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 919 msec 2024-11-07T12:42:34,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-11-07T12:42:34,034 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:42:34,035 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b2f185f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:42:34,035 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request db9ad1cb6cf9,38619,-1 for getting cluster id 2024-11-07T12:42:34,035 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:42:34,036 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e542096-4b0e-4b01-9952-662501fc0f65' 2024-11-07T12:42:34,036 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:42:34,036 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e542096-4b0e-4b01-9952-662501fc0f65" 2024-11-07T12:42:34,036 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bfa8d47, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:42:34,036 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [db9ad1cb6cf9,38619,-1] 2024-11-07T12:42:34,036 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:42:34,036 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:42:34,037 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50724, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:42:34,038 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4965d65b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:42:34,038 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a739f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:42:34,039 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request db9ad1cb6cf9,38619,-1 for getting cluster id 2024-11-07T12:42:34,039 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:42:34,039 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e542096-4b0e-4b01-9952-662501fc0f65' 2024-11-07T12:42:34,039 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:42:34,039 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e542096-4b0e-4b01-9952-662501fc0f65" 2024-11-07T12:42:34,040 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56fe3fb1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:42:34,040 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [db9ad1cb6cf9,38619,-1] 2024-11-07T12:42:34,040 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:42:34,040 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:42:34,041 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50746, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:42:34,041 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3edc9438, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:42:34,042 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@87fd48f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:42:34,042 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request db9ad1cb6cf9,38619,-1 for getting cluster id 2024-11-07T12:42:34,042 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:42:34,042 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e542096-4b0e-4b01-9952-662501fc0f65' 2024-11-07T12:42:34,043 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:42:34,043 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e542096-4b0e-4b01-9952-662501fc0f65" 2024-11-07T12:42:34,043 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@291eb0b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:42:34,043 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [db9ad1cb6cf9,38619,-1] 2024-11-07T12:42:34,043 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:42:34,043 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:42:34,044 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50756, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:42:34,044 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@585ce28f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:42:34,045 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f0bd6a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:42:34,045 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request db9ad1cb6cf9,38619,-1 for getting cluster id 2024-11-07T12:42:34,045 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:42:34,046 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e542096-4b0e-4b01-9952-662501fc0f65' 2024-11-07T12:42:34,046 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:42:34,046 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e542096-4b0e-4b01-9952-662501fc0f65" 2024-11-07T12:42:34,046 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@567d8054, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:42:34,046 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [db9ad1cb6cf9,38619,-1] 2024-11-07T12:42:34,046 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:42:34,046 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:42:34,047 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50778, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:42:34,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50eb99ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:42:34,048 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72c86b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:42:34,048 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request db9ad1cb6cf9,38619,-1 for getting cluster id 2024-11-07T12:42:34,049 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:42:34,049 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e542096-4b0e-4b01-9952-662501fc0f65' 2024-11-07T12:42:34,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:42:34,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e542096-4b0e-4b01-9952-662501fc0f65" 2024-11-07T12:42:34,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d80ebe7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:42:34,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [db9ad1cb6cf9,38619,-1] 2024-11-07T12:42:34,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:42:34,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:42:34,051 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50794, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:42:34,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70dea67, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:42:34,052 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23d098cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:42:34,052 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request db9ad1cb6cf9,38619,-1 for getting cluster id 2024-11-07T12:42:34,052 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:42:34,052 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e542096-4b0e-4b01-9952-662501fc0f65' 2024-11-07T12:42:34,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:42:34,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e542096-4b0e-4b01-9952-662501fc0f65" 2024-11-07T12:42:34,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18502a19, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:42:34,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [db9ad1cb6cf9,38619,-1] 2024-11-07T12:42:34,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:42:34,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:42:34,054 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50818, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:42:34,054 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fd6c868, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:42:34,055 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c9deaeb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:42:34,055 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request db9ad1cb6cf9,38619,-1 for getting cluster id 2024-11-07T12:42:34,055 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:42:34,056 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e542096-4b0e-4b01-9952-662501fc0f65' 2024-11-07T12:42:34,056 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:42:34,056 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e542096-4b0e-4b01-9952-662501fc0f65" 2024-11-07T12:42:34,056 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e63138d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:42:34,056 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [db9ad1cb6cf9,38619,-1] 2024-11-07T12:42:34,056 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:42:34,057 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:42:34,057 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50840, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:42:34,058 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d8dab24, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:42:34,058 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33f8ca2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:42:34,058 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request db9ad1cb6cf9,38619,-1 for getting cluster id 2024-11-07T12:42:34,058 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:42:34,059 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e542096-4b0e-4b01-9952-662501fc0f65' 2024-11-07T12:42:34,059 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:42:34,059 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e542096-4b0e-4b01-9952-662501fc0f65" 2024-11-07T12:42:34,059 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34ec6f20, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:42:34,059 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [db9ad1cb6cf9,38619,-1] 2024-11-07T12:42:34,059 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:42:34,060 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:42:34,060 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50868, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:42:34,061 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b89b31e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:42:34,061 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1515ed5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:42:34,061 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request db9ad1cb6cf9,38619,-1 for getting cluster id 2024-11-07T12:42:34,062 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:42:34,062 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e542096-4b0e-4b01-9952-662501fc0f65' 2024-11-07T12:42:34,062 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:42:34,062 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e542096-4b0e-4b01-9952-662501fc0f65" 2024-11-07T12:42:34,063 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ed4a744, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:42:34,063 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [db9ad1cb6cf9,38619,-1] 2024-11-07T12:42:34,063 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:42:34,063 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:42:34,064 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50898, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:42:34,064 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@624429ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:42:34,068 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d10c554, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:42:34,068 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request db9ad1cb6cf9,38619,-1 for getting cluster id 2024-11-07T12:42:34,068 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:42:34,068 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e542096-4b0e-4b01-9952-662501fc0f65' 2024-11-07T12:42:34,069 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:42:34,069 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e542096-4b0e-4b01-9952-662501fc0f65" 2024-11-07T12:42:34,069 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ae0f44a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:42:34,069 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [db9ad1cb6cf9,38619,-1] 2024-11-07T12:42:34,069 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:42:34,069 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:42:34,070 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50922, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:42:34,070 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f73116f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:42:34,072 DEBUG [Thread-1741 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:42:34,072 DEBUG [Thread-1743 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:42:34,072 DEBUG [Thread-1736 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:42:34,072 DEBUG [Thread-1730 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:42:34,072 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=-1] 2024-11-07T12:42:34,073 DEBUG [Thread-1739 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:42:34,073 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=-1] 2024-11-07T12:42:34,073 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:42:34,073 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:42:34,073 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=-1] 2024-11-07T12:42:34,073 DEBUG [Thread-1747 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:42:34,074 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55800, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:42:34,074 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55808, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:42:34,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=-1] 2024-11-07T12:42:34,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=-1] 2024-11-07T12:42:34,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:42:34,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:42:34,075 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=-1] 2024-11-07T12:42:34,075 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55812, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:42:34,075 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:42:34,075 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55828, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:42:34,076 DEBUG [Thread-1730 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:42:34,077 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5] 2024-11-07T12:42:34,077 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55844, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:42:34,077 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55852, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:42:34,077 DEBUG [Thread-1734 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:42:34,077 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5] 2024-11-07T12:42:34,079 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5] 2024-11-07T12:42:34,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5] 2024-11-07T12:42:34,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=-1] 2024-11-07T12:42:34,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5] 2024-11-07T12:42:34,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:42:34,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5] 2024-11-07T12:42:34,082 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:42:34,082 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55856, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:42:34,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-07T12:42:34,085 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:42:34,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-07T12:42:34,085 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:42:34,086 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:42:34,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5] 2024-11-07T12:42:34,089 DEBUG [Thread-1728 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:42:34,090 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=-1] 2024-11-07T12:42:34,090 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:42:34,091 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55858, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:42:34,092 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5] 2024-11-07T12:42:34,095 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ffbbe32019fd517ea21eab76e142aa7f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-07T12:42:34,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffbbe32019fd517ea21eab76e142aa7f, store=A 2024-11-07T12:42:34,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:34,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffbbe32019fd517ea21eab76e142aa7f, store=B 2024-11-07T12:42:34,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:34,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffbbe32019fd517ea21eab76e142aa7f, store=C 2024-11-07T12:42:34,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:34,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:34,097 DEBUG [Thread-1732 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:42:34,098 DEBUG [Thread-1732 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=-1] 2024-11-07T12:42:34,099 DEBUG [Thread-1732 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:42:34,100 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55862, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:42:34,107 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5] 2024-11-07T12:42:34,109 DEBUG [Thread-1745 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:42:34,109 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=-1] 2024-11-07T12:42:34,110 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:42:34,111 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55872, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:42:34,112 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5] 2024-11-07T12:42:34,139 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107cc5bbd79ce4d48f9bf5f502f3790f67a_ffbbe32019fd517ea21eab76e142aa7f is 50, key is test_row_0/A:col10/1730983354094/Put/seqid=0 2024-11-07T12:42:34,145 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1730983354121, max timestamp=1730983354141Num uniques 859; 2024-11-07T12:42:34,145 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-07T12:42:34,145 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1730983354121, max timestamp=1730983354141Num uniques 859; 2024-11-07T12:42:34,145 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-07T12:42:34,146 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1730983354121, max timestamp=1730983354141Num uniques 859; 2024-11-07T12:42:34,146 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-07T12:42:34,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742210_1386 (size=17034) 2024-11-07T12:42:34,183 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:34,188 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107cc5bbd79ce4d48f9bf5f502f3790f67a_ffbbe32019fd517ea21eab76e142aa7f to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107cc5bbd79ce4d48f9bf5f502f3790f67a_ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:34,189 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/dc3105dfbb8442f2aacd7173a87e4e25, store: [table=TestAcidGuarantees family=A region=ffbbe32019fd517ea21eab76e142aa7f] 2024-11-07T12:42:34,190 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/dc3105dfbb8442f2aacd7173a87e4e25 is 175, key is test_row_0/A:col10/1730983354094/Put/seqid=0 2024-11-07T12:42:34,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-07T12:42:34,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:34,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55862 deadline: 1730983414175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:34,196 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:34,197 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:34,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:34,197 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:34,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55858 deadline: 1730983414176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:34,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:34,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:34,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55844 deadline: 1730983414181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:34,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:34,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:34,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:34,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:34,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:34,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:34,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55856 deadline: 1730983414183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:34,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:34,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:34,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:34,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:34,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55828 deadline: 1730983414196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:34,202 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:34,202 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:34,202 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:34,239 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-07T12:42:34,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:34,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. as already flushing 2024-11-07T12:42:34,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:34,240 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:34,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:34,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:34,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742211_1387 (size=48139) 2024-11-07T12:42:34,254 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/dc3105dfbb8442f2aacd7173a87e4e25 2024-11-07T12:42:34,293 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/de9b805f732a4297a0e8f31367965300 is 50, key is test_row_0/B:col10/1730983354094/Put/seqid=0 2024-11-07T12:42:34,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742212_1388 (size=12001) 2024-11-07T12:42:34,307 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/de9b805f732a4297a0e8f31367965300 2024-11-07T12:42:34,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:34,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55862 deadline: 1730983414301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:34,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:34,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55856 deadline: 1730983414302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:34,313 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:34,313 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:34,313 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:34,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:34,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:34,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55844 deadline: 1730983414302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:34,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:34,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:34,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:34,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:34,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:34,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:34,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55858 deadline: 1730983414302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:34,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:34,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:34,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:34,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:34,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55828 deadline: 1730983414312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:34,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:34,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:34,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:34,353 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/4aaa81e78a504ad1be26b5dc7c1b4ba0 is 50, key is test_row_0/C:col10/1730983354094/Put/seqid=0 2024-11-07T12:42:34,393 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-07T12:42:34,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:34,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. as already flushing 2024-11-07T12:42:34,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:34,393 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:34,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:34,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:34,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-07T12:42:34,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742213_1389 (size=12001) 2024-11-07T12:42:34,421 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/4aaa81e78a504ad1be26b5dc7c1b4ba0 2024-11-07T12:42:34,435 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/dc3105dfbb8442f2aacd7173a87e4e25 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/dc3105dfbb8442f2aacd7173a87e4e25 2024-11-07T12:42:34,442 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/dc3105dfbb8442f2aacd7173a87e4e25, entries=250, sequenceid=15, filesize=47.0 K 2024-11-07T12:42:34,444 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/de9b805f732a4297a0e8f31367965300 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/de9b805f732a4297a0e8f31367965300 2024-11-07T12:42:34,451 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/de9b805f732a4297a0e8f31367965300, entries=150, sequenceid=15, filesize=11.7 K 2024-11-07T12:42:34,455 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/4aaa81e78a504ad1be26b5dc7c1b4ba0 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/4aaa81e78a504ad1be26b5dc7c1b4ba0 2024-11-07T12:42:34,459 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/4aaa81e78a504ad1be26b5dc7c1b4ba0, entries=150, sequenceid=15, filesize=11.7 K 2024-11-07T12:42:34,461 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for ffbbe32019fd517ea21eab76e142aa7f in 365ms, sequenceid=15, compaction requested=false 2024-11-07T12:42:34,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ffbbe32019fd517ea21eab76e142aa7f: 2024-11-07T12:42:34,531 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ffbbe32019fd517ea21eab76e142aa7f 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-07T12:42:34,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffbbe32019fd517ea21eab76e142aa7f, store=A 2024-11-07T12:42:34,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:42:34,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffbbe32019fd517ea21eab76e142aa7f, store=B 2024-11-07T12:42:34,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:42:34,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffbbe32019fd517ea21eab76e142aa7f, store=C 2024-11-07T12:42:34,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:42:34,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:34,547 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-07T12:42:34,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:34,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. as already flushing 2024-11-07T12:42:34,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:34,548 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:34,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:34,549 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110730340b52b2b8432d8ea79f0925a5ce86_ffbbe32019fd517ea21eab76e142aa7f is 50, key is test_row_0/A:col10/1730983354529/Put/seqid=0 2024-11-07T12:42:34,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:34,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:34,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55856 deadline: 1730983414540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:34,560 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:34,560 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:34,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:34,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:34,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55828 deadline: 1730983414541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:34,563 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:34,563 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:34,563 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:34,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742214_1390 (size=14594) 2024-11-07T12:42:34,573 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:34,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:34,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55858 deadline: 1730983414561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:34,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:34,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:34,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:34,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:34,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55844 deadline: 1730983414561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:34,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:34,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:34,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:34,581 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110730340b52b2b8432d8ea79f0925a5ce86_ffbbe32019fd517ea21eab76e142aa7f to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110730340b52b2b8432d8ea79f0925a5ce86_ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:34,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:34,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55862 deadline: 1730983414562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:34,583 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:34,583 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:34,583 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:34,584 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/de98c2784445452dabe5016d7eb58d9a, store: [table=TestAcidGuarantees family=A region=ffbbe32019fd517ea21eab76e142aa7f] 2024-11-07T12:42:34,586 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/de98c2784445452dabe5016d7eb58d9a is 175, key is test_row_0/A:col10/1730983354529/Put/seqid=0 2024-11-07T12:42:34,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742215_1391 (size=39549) 2024-11-07T12:42:34,595 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/de98c2784445452dabe5016d7eb58d9a 2024-11-07T12:42:34,609 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/f2f002b36f204464820f32bccb430f65 is 50, key is test_row_0/B:col10/1730983354529/Put/seqid=0 2024-11-07T12:42:34,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742216_1392 (size=12001) 2024-11-07T12:42:34,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:34,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55856 deadline: 1730983414671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:34,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:34,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:34,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:34,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55828 deadline: 1730983414672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:34,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:34,675 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:34,675 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:34,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:34,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:34,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55858 deadline: 1730983414681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:34,693 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:34,693 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:34,693 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:34,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:34,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55844 deadline: 1730983414692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:34,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:34,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:34,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:34,701 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-07T12:42:34,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:34,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. as already flushing 2024-11-07T12:42:34,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:34,702 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:34,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:34,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:34,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:34,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55862 deadline: 1730983414694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:34,705 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:34,706 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:34,706 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:34,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-07T12:42:34,855 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-07T12:42:34,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:34,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. as already flushing 2024-11-07T12:42:34,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:34,856 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:34,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:34,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:34,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:34,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55856 deadline: 1730983414882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:34,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:34,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:34,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:34,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:34,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55828 deadline: 1730983414882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:34,883 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:34,883 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:34,883 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:34,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:34,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55858 deadline: 1730983414902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:34,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:34,909 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:34,909 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:34,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55844 deadline: 1730983414902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:34,909 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:34,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:34,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:34,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:34,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:34,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55862 deadline: 1730983414911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:34,924 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:34,924 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:34,924 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:35,009 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-07T12:42:35,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:35,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. as already flushing 2024-11-07T12:42:35,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:35,010 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:35,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:35,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:35,021 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/f2f002b36f204464820f32bccb430f65 2024-11-07T12:42:35,029 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/07ffca961d214d1f91d762d81dc0bec4 is 50, key is test_row_0/C:col10/1730983354529/Put/seqid=0 2024-11-07T12:42:35,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742217_1393 (size=12001) 2024-11-07T12:42:35,036 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/07ffca961d214d1f91d762d81dc0bec4 2024-11-07T12:42:35,040 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/de98c2784445452dabe5016d7eb58d9a as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/de98c2784445452dabe5016d7eb58d9a 2024-11-07T12:42:35,049 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/de98c2784445452dabe5016d7eb58d9a, entries=200, sequenceid=50, filesize=38.6 K 2024-11-07T12:42:35,050 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/f2f002b36f204464820f32bccb430f65 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/f2f002b36f204464820f32bccb430f65 2024-11-07T12:42:35,062 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/f2f002b36f204464820f32bccb430f65, entries=150, sequenceid=50, filesize=11.7 K 2024-11-07T12:42:35,062 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/07ffca961d214d1f91d762d81dc0bec4 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/07ffca961d214d1f91d762d81dc0bec4 2024-11-07T12:42:35,071 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/07ffca961d214d1f91d762d81dc0bec4, entries=150, sequenceid=50, filesize=11.7 K 2024-11-07T12:42:35,072 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for ffbbe32019fd517ea21eab76e142aa7f in 541ms, sequenceid=50, compaction requested=false 2024-11-07T12:42:35,072 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ffbbe32019fd517ea21eab76e142aa7f: 2024-11-07T12:42:35,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-07T12:42:35,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:35,163 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing ffbbe32019fd517ea21eab76e142aa7f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-07T12:42:35,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffbbe32019fd517ea21eab76e142aa7f, store=A 2024-11-07T12:42:35,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:35,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffbbe32019fd517ea21eab76e142aa7f, store=B 2024-11-07T12:42:35,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:35,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffbbe32019fd517ea21eab76e142aa7f, store=C 2024-11-07T12:42:35,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:35,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411072bdab52aba86487799c64cbd22af82ee_ffbbe32019fd517ea21eab76e142aa7f is 50, key is test_row_0/A:col10/1730983354534/Put/seqid=0 2024-11-07T12:42:35,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742218_1394 (size=12154) 2024-11-07T12:42:35,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:35,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. as already flushing 2024-11-07T12:42:35,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-07T12:42:35,290 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1730983355242, max timestamp=1730983355271Num uniques 859; 2024-11-07T12:42:35,290 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-07T12:42:35,290 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1730983355242, max timestamp=1730983355271Num uniques 859; 2024-11-07T12:42:35,290 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-07T12:42:35,291 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1730983355242, max timestamp=1730983355271Num uniques 859; 2024-11-07T12:42:35,291 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-07T12:42:35,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:35,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55828 deadline: 1730983415337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:35,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:35,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55856 deadline: 1730983415337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:35,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:35,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55862 deadline: 1730983415338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:35,361 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:35,362 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:35,362 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:35,362 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:35,362 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:35,362 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:35,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:35,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:35,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:35,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:35,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55844 deadline: 1730983415360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:35,376 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:35,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:35,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:35,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55858 deadline: 1730983415361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:35,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:35,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:35,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:35,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:35,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:35,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:35,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55862 deadline: 1730983415472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:35,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55856 deadline: 1730983415472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:35,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:35,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55828 deadline: 1730983415472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:35,488 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:35,488 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:35,511 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:35,511 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:35,511 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:35,511 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:35,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:35,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:35,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:35,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:35,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55844 deadline: 1730983415481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:35,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:35,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:35,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:35,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:35,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55858 deadline: 1730983415481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:35,516 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:35,516 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:35,516 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:35,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:35,600 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411072bdab52aba86487799c64cbd22af82ee_ffbbe32019fd517ea21eab76e142aa7f to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411072bdab52aba86487799c64cbd22af82ee_ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:35,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/3500be1a50634c20ada4f06730f47d8e, store: [table=TestAcidGuarantees family=A region=ffbbe32019fd517ea21eab76e142aa7f] 2024-11-07T12:42:35,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/3500be1a50634c20ada4f06730f47d8e is 175, key is test_row_0/A:col10/1730983354534/Put/seqid=0 2024-11-07T12:42:35,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742219_1395 (size=30955) 2024-11-07T12:42:35,607 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=60, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/3500be1a50634c20ada4f06730f47d8e 2024-11-07T12:42:35,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/ad161989282d40e282c6622f40da2d0b is 50, key is test_row_0/B:col10/1730983354534/Put/seqid=0 2024-11-07T12:42:35,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742220_1396 (size=12001) 2024-11-07T12:42:35,619 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/ad161989282d40e282c6622f40da2d0b 2024-11-07T12:42:35,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/970c989c5c944fd3bb2a20b812f00291 is 50, key is test_row_0/C:col10/1730983354534/Put/seqid=0 2024-11-07T12:42:35,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742221_1397 (size=12001) 2024-11-07T12:42:35,634 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/970c989c5c944fd3bb2a20b812f00291 2024-11-07T12:42:35,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/3500be1a50634c20ada4f06730f47d8e as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/3500be1a50634c20ada4f06730f47d8e 2024-11-07T12:42:35,646 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/3500be1a50634c20ada4f06730f47d8e, entries=150, sequenceid=60, filesize=30.2 K 2024-11-07T12:42:35,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/ad161989282d40e282c6622f40da2d0b as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/ad161989282d40e282c6622f40da2d0b 2024-11-07T12:42:35,650 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/ad161989282d40e282c6622f40da2d0b, entries=150, sequenceid=60, filesize=11.7 K 2024-11-07T12:42:35,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/970c989c5c944fd3bb2a20b812f00291 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/970c989c5c944fd3bb2a20b812f00291 2024-11-07T12:42:35,656 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/970c989c5c944fd3bb2a20b812f00291, entries=150, sequenceid=60, filesize=11.7 K 2024-11-07T12:42:35,657 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=136.02 KB/139286 for ffbbe32019fd517ea21eab76e142aa7f in 494ms, sequenceid=60, compaction requested=true 2024-11-07T12:42:35,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for ffbbe32019fd517ea21eab76e142aa7f: 2024-11-07T12:42:35,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:35,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-07T12:42:35,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-07T12:42:35,661 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-07T12:42:35,661 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5740 sec 2024-11-07T12:42:35,664 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 1.5800 sec 2024-11-07T12:42:35,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:35,728 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ffbbe32019fd517ea21eab76e142aa7f 3/3 column families, dataSize=142.73 KB heapSize=404.38 KB 2024-11-07T12:42:35,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffbbe32019fd517ea21eab76e142aa7f, store=A 2024-11-07T12:42:35,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:42:35,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffbbe32019fd517ea21eab76e142aa7f, store=B 2024-11-07T12:42:35,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:42:35,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffbbe32019fd517ea21eab76e142aa7f, store=C 2024-11-07T12:42:35,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:42:35,744 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110772b5580f9aef4b2b96136ef4ecb43435_ffbbe32019fd517ea21eab76e142aa7f is 50, key is test_row_0/A:col10/1730983355727/Put/seqid=0 2024-11-07T12:42:35,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:35,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55828 deadline: 1730983415733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:35,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:35,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55844 deadline: 1730983415735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:35,751 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:35,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:35,751 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:35,751 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:35,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55862 deadline: 1730983415741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:35,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:35,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:35,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:35,751 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:35,751 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:35,751 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:35,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742222_1398 (size=14594) 2024-11-07T12:42:35,755 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:35,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:35,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55856 deadline: 1730983415751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:35,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:35,758 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:35,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:35,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55858 deadline: 1730983415751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:35,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:35,759 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:35,759 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:35,759 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:35,761 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110772b5580f9aef4b2b96136ef4ecb43435_ffbbe32019fd517ea21eab76e142aa7f to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110772b5580f9aef4b2b96136ef4ecb43435_ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:35,762 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/54228504239146e5b1cd5b0d3709c1e6, store: [table=TestAcidGuarantees family=A region=ffbbe32019fd517ea21eab76e142aa7f] 2024-11-07T12:42:35,763 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/54228504239146e5b1cd5b0d3709c1e6 is 175, key is test_row_0/A:col10/1730983355727/Put/seqid=0 2024-11-07T12:42:35,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742223_1399 (size=39549) 2024-11-07T12:42:35,776 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=97, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/54228504239146e5b1cd5b0d3709c1e6 2024-11-07T12:42:35,785 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/bac4e360b16840bc9f3193ac02236cf3 is 50, key is test_row_0/B:col10/1730983355727/Put/seqid=0 2024-11-07T12:42:35,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742224_1400 (size=12001) 2024-11-07T12:42:35,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:35,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55844 deadline: 1730983415864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:35,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:35,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:35,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:35,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:35,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55828 deadline: 1730983415864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:35,870 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:35,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:35,870 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:35,871 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:35,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55858 deadline: 1730983415865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:35,871 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:35,871 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:35,871 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:35,871 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:35,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55862 deadline: 1730983415865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:35,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:35,871 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:35,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:35,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:35,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55856 deadline: 1730983415867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:35,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:35,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:35,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:36,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:36,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55844 deadline: 1730983416072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:36,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:36,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:36,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:36,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:36,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55862 deadline: 1730983416082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:36,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:36,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55858 deadline: 1730983416082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:36,097 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:36,098 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:36,098 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:36,100 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:36,101 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:36,101 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:36,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:36,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55828 deadline: 1730983416082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:36,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:36,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:36,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:36,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55856 deadline: 1730983416082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:36,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:36,102 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:36,102 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:36,102 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:36,195 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/bac4e360b16840bc9f3193ac02236cf3 2024-11-07T12:42:36,208 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/e782f5db518149b9a3d6e5f4e7efaca1 is 50, key is test_row_0/C:col10/1730983355727/Put/seqid=0 2024-11-07T12:42:36,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742225_1401 (size=12001) 2024-11-07T12:42:36,215 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.33 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/e782f5db518149b9a3d6e5f4e7efaca1 2024-11-07T12:42:36,222 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/54228504239146e5b1cd5b0d3709c1e6 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/54228504239146e5b1cd5b0d3709c1e6 2024-11-07T12:42:36,225 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/54228504239146e5b1cd5b0d3709c1e6, entries=200, sequenceid=97, filesize=38.6 K 2024-11-07T12:42:36,226 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/bac4e360b16840bc9f3193ac02236cf3 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/bac4e360b16840bc9f3193ac02236cf3 2024-11-07T12:42:36,229 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/bac4e360b16840bc9f3193ac02236cf3, entries=150, sequenceid=97, filesize=11.7 K 2024-11-07T12:42:36,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/e782f5db518149b9a3d6e5f4e7efaca1 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/e782f5db518149b9a3d6e5f4e7efaca1 2024-11-07T12:42:36,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-07T12:42:36,232 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:42:36,233 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:42:36,233 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/e782f5db518149b9a3d6e5f4e7efaca1, entries=150, sequenceid=97, filesize=11.7 K 2024-11-07T12:42:36,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-07T12:42:36,234 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~142.73 KB/146156, heapSize ~404.34 KB/414040, currentSize=46.96 KB/48090 for ffbbe32019fd517ea21eab76e142aa7f in 506ms, sequenceid=97, compaction requested=true 2024-11-07T12:42:36,234 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ffbbe32019fd517ea21eab76e142aa7f: 2024-11-07T12:42:36,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffbbe32019fd517ea21eab76e142aa7f:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:42:36,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:42:36,234 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:42:36,235 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:42:36,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffbbe32019fd517ea21eab76e142aa7f:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:42:36,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:42:36,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffbbe32019fd517ea21eab76e142aa7f:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:42:36,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:42:36,235 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:42:36,237 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:42:36,237 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:42:36,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-07T12:42:36,240 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:42:36,240 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 158192 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:42:36,240 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): ffbbe32019fd517ea21eab76e142aa7f/A is initiating minor compaction (all files) 2024-11-07T12:42:36,240 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffbbe32019fd517ea21eab76e142aa7f/A in TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:36,241 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/dc3105dfbb8442f2aacd7173a87e4e25, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/de98c2784445452dabe5016d7eb58d9a, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/3500be1a50634c20ada4f06730f47d8e, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/54228504239146e5b1cd5b0d3709c1e6] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp, totalSize=154.5 K 2024-11-07T12:42:36,241 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:36,241 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. files: [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/dc3105dfbb8442f2aacd7173a87e4e25, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/de98c2784445452dabe5016d7eb58d9a, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/3500be1a50634c20ada4f06730f47d8e, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/54228504239146e5b1cd5b0d3709c1e6] 2024-11-07T12:42:36,241 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): ffbbe32019fd517ea21eab76e142aa7f/B is initiating minor compaction (all files) 2024-11-07T12:42:36,241 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffbbe32019fd517ea21eab76e142aa7f/B in TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:36,241 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/de9b805f732a4297a0e8f31367965300, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/f2f002b36f204464820f32bccb430f65, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/ad161989282d40e282c6622f40da2d0b, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/bac4e360b16840bc9f3193ac02236cf3] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp, totalSize=46.9 K 2024-11-07T12:42:36,241 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc3105dfbb8442f2aacd7173a87e4e25, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1730983354087 2024-11-07T12:42:36,241 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting de9b805f732a4297a0e8f31367965300, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1730983354090 2024-11-07T12:42:36,242 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting de98c2784445452dabe5016d7eb58d9a, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1730983354179 2024-11-07T12:42:36,242 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3500be1a50634c20ada4f06730f47d8e, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1730983354534 2024-11-07T12:42:36,242 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting f2f002b36f204464820f32bccb430f65, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1730983354180 2024-11-07T12:42:36,242 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting ad161989282d40e282c6622f40da2d0b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1730983354534 2024-11-07T12:42:36,242 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54228504239146e5b1cd5b0d3709c1e6, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1730983355329 2024-11-07T12:42:36,243 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting bac4e360b16840bc9f3193ac02236cf3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1730983355329 2024-11-07T12:42:36,252 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ffbbe32019fd517ea21eab76e142aa7f] 2024-11-07T12:42:36,267 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffbbe32019fd517ea21eab76e142aa7f#B#compaction#346 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:42:36,268 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/82c0b841a3b04d40a87dc1dd14832e6c is 50, key is test_row_0/B:col10/1730983355727/Put/seqid=0 2024-11-07T12:42:36,275 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241107d6b25252395e47e38c53aee7cd0ca769_ffbbe32019fd517ea21eab76e142aa7f store=[table=TestAcidGuarantees family=A region=ffbbe32019fd517ea21eab76e142aa7f] 2024-11-07T12:42:36,282 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241107d6b25252395e47e38c53aee7cd0ca769_ffbbe32019fd517ea21eab76e142aa7f, store=[table=TestAcidGuarantees family=A region=ffbbe32019fd517ea21eab76e142aa7f] 2024-11-07T12:42:36,282 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107d6b25252395e47e38c53aee7cd0ca769_ffbbe32019fd517ea21eab76e142aa7f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ffbbe32019fd517ea21eab76e142aa7f] 2024-11-07T12:42:36,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742226_1402 (size=12139) 2024-11-07T12:42:36,316 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/82c0b841a3b04d40a87dc1dd14832e6c as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/82c0b841a3b04d40a87dc1dd14832e6c 2024-11-07T12:42:36,328 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ffbbe32019fd517ea21eab76e142aa7f/B of ffbbe32019fd517ea21eab76e142aa7f into 82c0b841a3b04d40a87dc1dd14832e6c(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:42:36,328 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffbbe32019fd517ea21eab76e142aa7f: 2024-11-07T12:42:36,328 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., storeName=ffbbe32019fd517ea21eab76e142aa7f/B, priority=12, startTime=1730983356234; duration=0sec 2024-11-07T12:42:36,330 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:42:36,330 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffbbe32019fd517ea21eab76e142aa7f:B 2024-11-07T12:42:36,330 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:42:36,333 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:42:36,333 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): ffbbe32019fd517ea21eab76e142aa7f/C is initiating minor compaction (all files) 2024-11-07T12:42:36,333 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffbbe32019fd517ea21eab76e142aa7f/C in TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:36,333 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/4aaa81e78a504ad1be26b5dc7c1b4ba0, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/07ffca961d214d1f91d762d81dc0bec4, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/970c989c5c944fd3bb2a20b812f00291, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/e782f5db518149b9a3d6e5f4e7efaca1] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp, totalSize=46.9 K 2024-11-07T12:42:36,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742227_1403 (size=4469) 2024-11-07T12:42:36,334 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 4aaa81e78a504ad1be26b5dc7c1b4ba0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1730983354090 2024-11-07T12:42:36,334 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 07ffca961d214d1f91d762d81dc0bec4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1730983354180 2024-11-07T12:42:36,335 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 970c989c5c944fd3bb2a20b812f00291, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1730983354534 2024-11-07T12:42:36,335 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting e782f5db518149b9a3d6e5f4e7efaca1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1730983355329 2024-11-07T12:42:36,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-07T12:42:36,346 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffbbe32019fd517ea21eab76e142aa7f#C#compaction#347 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:42:36,352 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T12:42:36,352 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/c74ce60b8291448ba153fbfb20c6fba4 is 50, key is test_row_0/C:col10/1730983355727/Put/seqid=0 2024-11-07T12:42:36,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742228_1404 (size=12139) 2024-11-07T12:42:36,391 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-07T12:42:36,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:36,391 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing ffbbe32019fd517ea21eab76e142aa7f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-07T12:42:36,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffbbe32019fd517ea21eab76e142aa7f, store=A 2024-11-07T12:42:36,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:36,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffbbe32019fd517ea21eab76e142aa7f, store=B 2024-11-07T12:42:36,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:36,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffbbe32019fd517ea21eab76e142aa7f, store=C 2024-11-07T12:42:36,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:36,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:36,403 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. as already flushing 2024-11-07T12:42:36,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107092d7e369cc748faad7a156de2f565dd_ffbbe32019fd517ea21eab76e142aa7f is 50, key is test_row_0/A:col10/1730983355749/Put/seqid=0 2024-11-07T12:42:36,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742229_1405 (size=12154) 2024-11-07T12:42:36,462 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1730983356437, max timestamp=1730983356458Num uniques 859; 2024-11-07T12:42:36,462 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1730983356437, max timestamp=1730983356458Num uniques 859; 2024-11-07T12:42:36,462 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-07T12:42:36,462 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-07T12:42:36,462 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1730983356437, max timestamp=1730983356458Num uniques 859; 2024-11-07T12:42:36,462 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-07T12:42:36,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:36,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55858 deadline: 1730983416494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:36,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:36,508 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:36,508 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:36,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:36,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:36,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55856 deadline: 1730983416496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:36,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55862 deadline: 1730983416497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:36,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:36,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55844 deadline: 1730983416506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:36,519 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:36,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:36,519 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:36,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:36,519 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:36,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:36,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:36,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:36,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:36,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55828 deadline: 1730983416507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:36,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:36,520 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:36,520 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:36,520 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:36,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-07T12:42:36,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:36,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55858 deadline: 1730983416612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:36,628 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:36,628 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:36,628 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:36,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:36,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55862 deadline: 1730983416621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:36,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:36,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55856 deadline: 1730983416622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:36,633 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:36,633 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:36,633 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:36,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:36,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55828 deadline: 1730983416622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:36,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:36,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:36,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:36,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:36,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55844 deadline: 1730983416622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:36,634 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:36,634 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:36,634 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:36,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:36,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:36,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:36,744 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffbbe32019fd517ea21eab76e142aa7f#A#compaction#345 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:42:36,745 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/0dfb744c68ad4fb98ed8363fb434ba1b is 175, key is test_row_0/A:col10/1730983355727/Put/seqid=0 2024-11-07T12:42:36,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742230_1406 (size=31093) 2024-11-07T12:42:36,788 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/c74ce60b8291448ba153fbfb20c6fba4 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/c74ce60b8291448ba153fbfb20c6fba4 2024-11-07T12:42:36,794 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ffbbe32019fd517ea21eab76e142aa7f/C of ffbbe32019fd517ea21eab76e142aa7f into c74ce60b8291448ba153fbfb20c6fba4(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:42:36,794 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffbbe32019fd517ea21eab76e142aa7f: 2024-11-07T12:42:36,794 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., storeName=ffbbe32019fd517ea21eab76e142aa7f/C, priority=12, startTime=1730983356235; duration=0sec 2024-11-07T12:42:36,794 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:42:36,794 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffbbe32019fd517ea21eab76e142aa7f:C 2024-11-07T12:42:36,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:36,830 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107092d7e369cc748faad7a156de2f565dd_ffbbe32019fd517ea21eab76e142aa7f to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107092d7e369cc748faad7a156de2f565dd_ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:36,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/a84c8dae672c4f99a58382f672f3af10, store: [table=TestAcidGuarantees family=A region=ffbbe32019fd517ea21eab76e142aa7f] 2024-11-07T12:42:36,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/a84c8dae672c4f99a58382f672f3af10 is 175, key is test_row_0/A:col10/1730983355749/Put/seqid=0 2024-11-07T12:42:36,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742231_1407 (size=30955) 2024-11-07T12:42:36,839 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=108, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/a84c8dae672c4f99a58382f672f3af10 2024-11-07T12:42:36,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:36,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55858 deadline: 1730983416832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:36,843 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:36,843 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:36,843 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:36,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/44e4b025cfdc4ce299d491a6a515315f is 50, key is test_row_0/B:col10/1730983355749/Put/seqid=0 2024-11-07T12:42:36,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:36,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55862 deadline: 1730983416842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:36,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:36,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:36,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:36,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55856 deadline: 1730983416842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:36,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:36,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:36,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:36,854 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:36,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:36,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55828 deadline: 1730983416842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:36,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:36,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:36,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:36,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742232_1408 (size=12001) 2024-11-07T12:42:36,858 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/44e4b025cfdc4ce299d491a6a515315f 2024-11-07T12:42:36,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:36,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55844 deadline: 1730983416843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:36,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:36,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:36,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:36,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-07T12:42:36,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/465ff595d6e34831b549dddba14c03e1 is 50, key is test_row_0/C:col10/1730983355749/Put/seqid=0 2024-11-07T12:42:36,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742233_1409 (size=12001) 2024-11-07T12:42:36,886 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/465ff595d6e34831b549dddba14c03e1 2024-11-07T12:42:36,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/a84c8dae672c4f99a58382f672f3af10 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/a84c8dae672c4f99a58382f672f3af10 2024-11-07T12:42:36,900 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/a84c8dae672c4f99a58382f672f3af10, entries=150, sequenceid=108, filesize=30.2 K 2024-11-07T12:42:36,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/44e4b025cfdc4ce299d491a6a515315f as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/44e4b025cfdc4ce299d491a6a515315f 2024-11-07T12:42:36,905 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/44e4b025cfdc4ce299d491a6a515315f, entries=150, sequenceid=108, filesize=11.7 K 2024-11-07T12:42:36,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/465ff595d6e34831b549dddba14c03e1 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/465ff595d6e34831b549dddba14c03e1 2024-11-07T12:42:36,912 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/465ff595d6e34831b549dddba14c03e1, entries=150, sequenceid=108, filesize=11.7 K 2024-11-07T12:42:36,913 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for ffbbe32019fd517ea21eab76e142aa7f in 522ms, sequenceid=108, compaction requested=false 2024-11-07T12:42:36,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for ffbbe32019fd517ea21eab76e142aa7f: 2024-11-07T12:42:36,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:36,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-07T12:42:36,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-07T12:42:36,917 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-07T12:42:36,917 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 678 msec 2024-11-07T12:42:36,922 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 685 msec 2024-11-07T12:42:37,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:37,164 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ffbbe32019fd517ea21eab76e142aa7f 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-11-07T12:42:37,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffbbe32019fd517ea21eab76e142aa7f, store=A 2024-11-07T12:42:37,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:42:37,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffbbe32019fd517ea21eab76e142aa7f, store=B 2024-11-07T12:42:37,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:42:37,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffbbe32019fd517ea21eab76e142aa7f, store=C 2024-11-07T12:42:37,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:42:37,169 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/0dfb744c68ad4fb98ed8363fb434ba1b as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/0dfb744c68ad4fb98ed8363fb434ba1b 2024-11-07T12:42:37,175 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110754796cc89aeb4d92920e080c75ea1e6a_ffbbe32019fd517ea21eab76e142aa7f is 50, key is test_row_0/A:col10/1730983357161/Put/seqid=0 2024-11-07T12:42:37,177 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ffbbe32019fd517ea21eab76e142aa7f/A of ffbbe32019fd517ea21eab76e142aa7f into 0dfb744c68ad4fb98ed8363fb434ba1b(size=30.4 K), total size for store is 60.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:42:37,177 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffbbe32019fd517ea21eab76e142aa7f: 2024-11-07T12:42:37,177 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., storeName=ffbbe32019fd517ea21eab76e142aa7f/A, priority=12, startTime=1730983356234; duration=0sec 2024-11-07T12:42:37,177 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:42:37,177 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffbbe32019fd517ea21eab76e142aa7f:A 2024-11-07T12:42:37,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742234_1410 (size=14794) 2024-11-07T12:42:37,183 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:37,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:37,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55828 deadline: 1730983417172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:37,187 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110754796cc89aeb4d92920e080c75ea1e6a_ffbbe32019fd517ea21eab76e142aa7f to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110754796cc89aeb4d92920e080c75ea1e6a_ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:37,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:37,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:37,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:37,191 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/26bc571ba9af40b783df4847b1de8007, store: [table=TestAcidGuarantees family=A region=ffbbe32019fd517ea21eab76e142aa7f] 2024-11-07T12:42:37,191 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/26bc571ba9af40b783df4847b1de8007 is 175, key is test_row_0/A:col10/1730983357161/Put/seqid=0 2024-11-07T12:42:37,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:37,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55862 deadline: 1730983417178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:37,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:37,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55858 deadline: 1730983417182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:37,195 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:37,195 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:37,195 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:37,196 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:37,196 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:37,196 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:37,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:37,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55856 deadline: 1730983417183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:37,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:37,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55844 deadline: 1730983417185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:37,203 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:37,204 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:37,205 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:37,205 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:37,205 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:37,205 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:37,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742235_1411 (size=39749) 2024-11-07T12:42:37,232 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=145, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/26bc571ba9af40b783df4847b1de8007 2024-11-07T12:42:37,256 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/ac74714f4c604abaa8e2015d1ba05cd0 is 50, key is test_row_0/B:col10/1730983357161/Put/seqid=0 2024-11-07T12:42:37,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742236_1412 (size=12151) 2024-11-07T12:42:37,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:37,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55828 deadline: 1730983417292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:37,304 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:37,304 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:37,304 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:37,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:37,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55858 deadline: 1730983417302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:37,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:37,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:37,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:37,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:37,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55862 deadline: 1730983417302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:37,314 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:37,314 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:37,314 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:37,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:37,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55844 deadline: 1730983417311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:37,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:37,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:37,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:37,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:37,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55856 deadline: 1730983417311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:37,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:37,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:37,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:37,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-07T12:42:37,372 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:42:37,373 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:42:37,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-07T12:42:37,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-07T12:42:37,375 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:42:37,375 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:42:37,376 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:42:37,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-07T12:42:37,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:37,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55828 deadline: 1730983417511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:37,525 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:37,525 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:37,525 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:37,528 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-07T12:42:37,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:37,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. as already flushing 2024-11-07T12:42:37,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:37,529 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:37,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:37,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:37,533 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:37,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55858 deadline: 1730983417522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:37,533 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:37,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:37,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55862 deadline: 1730983417523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:37,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:37,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:37,535 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:37,536 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:37,536 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:37,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:37,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55844 deadline: 1730983417536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:37,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:37,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55856 deadline: 1730983417538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:37,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:37,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:37,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:37,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:37,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:37,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:37,671 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/ac74714f4c604abaa8e2015d1ba05cd0 2024-11-07T12:42:37,683 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/c2d1ad17ea484d5bb8c61d4da470e282 is 50, key is test_row_0/C:col10/1730983357161/Put/seqid=0 2024-11-07T12:42:37,687 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-07T12:42:37,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:37,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. as already flushing 2024-11-07T12:42:37,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:37,687 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:37,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:37,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:37,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-07T12:42:37,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742237_1413 (size=12151) 2024-11-07T12:42:37,731 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/c2d1ad17ea484d5bb8c61d4da470e282 2024-11-07T12:42:37,743 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/26bc571ba9af40b783df4847b1de8007 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/26bc571ba9af40b783df4847b1de8007 2024-11-07T12:42:37,752 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/26bc571ba9af40b783df4847b1de8007, entries=200, sequenceid=145, filesize=38.8 K 2024-11-07T12:42:37,753 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/ac74714f4c604abaa8e2015d1ba05cd0 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/ac74714f4c604abaa8e2015d1ba05cd0 2024-11-07T12:42:37,758 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/ac74714f4c604abaa8e2015d1ba05cd0, entries=150, sequenceid=145, filesize=11.9 K 2024-11-07T12:42:37,761 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/c2d1ad17ea484d5bb8c61d4da470e282 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/c2d1ad17ea484d5bb8c61d4da470e282 2024-11-07T12:42:37,779 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/c2d1ad17ea484d5bb8c61d4da470e282, entries=150, sequenceid=145, filesize=11.9 K 2024-11-07T12:42:37,780 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=40.25 KB/41220 for ffbbe32019fd517ea21eab76e142aa7f in 615ms, sequenceid=145, compaction requested=true 2024-11-07T12:42:37,780 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ffbbe32019fd517ea21eab76e142aa7f: 2024-11-07T12:42:37,780 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:42:37,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffbbe32019fd517ea21eab76e142aa7f:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:42:37,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:42:37,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffbbe32019fd517ea21eab76e142aa7f:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:42:37,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:42:37,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffbbe32019fd517ea21eab76e142aa7f:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:42:37,780 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:42:37,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:42:37,781 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101797 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:42:37,781 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): ffbbe32019fd517ea21eab76e142aa7f/A is initiating minor compaction (all files) 2024-11-07T12:42:37,781 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffbbe32019fd517ea21eab76e142aa7f/A in TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:37,781 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/0dfb744c68ad4fb98ed8363fb434ba1b, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/a84c8dae672c4f99a58382f672f3af10, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/26bc571ba9af40b783df4847b1de8007] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp, totalSize=99.4 K 2024-11-07T12:42:37,781 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:37,781 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. files: [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/0dfb744c68ad4fb98ed8363fb434ba1b, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/a84c8dae672c4f99a58382f672f3af10, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/26bc571ba9af40b783df4847b1de8007] 2024-11-07T12:42:37,781 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36291 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:42:37,781 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): ffbbe32019fd517ea21eab76e142aa7f/B is initiating minor compaction (all files) 2024-11-07T12:42:37,781 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffbbe32019fd517ea21eab76e142aa7f/B in TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:37,781 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/82c0b841a3b04d40a87dc1dd14832e6c, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/44e4b025cfdc4ce299d491a6a515315f, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/ac74714f4c604abaa8e2015d1ba05cd0] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp, totalSize=35.4 K 2024-11-07T12:42:37,782 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0dfb744c68ad4fb98ed8363fb434ba1b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1730983355329 2024-11-07T12:42:37,782 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 82c0b841a3b04d40a87dc1dd14832e6c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1730983355329 2024-11-07T12:42:37,782 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting a84c8dae672c4f99a58382f672f3af10, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1730983355730 2024-11-07T12:42:37,782 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 44e4b025cfdc4ce299d491a6a515315f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1730983355730 2024-11-07T12:42:37,783 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 26bc571ba9af40b783df4847b1de8007, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1730983356494 2024-11-07T12:42:37,783 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting ac74714f4c604abaa8e2015d1ba05cd0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1730983356494 2024-11-07T12:42:37,791 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ffbbe32019fd517ea21eab76e142aa7f] 2024-11-07T12:42:37,795 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffbbe32019fd517ea21eab76e142aa7f#B#compaction#355 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:42:37,795 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/c36e10399d0047bcbf6f1e69fe05ab5b is 50, key is test_row_0/B:col10/1730983357161/Put/seqid=0 2024-11-07T12:42:37,797 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241107847a43c2b01a47e1af75f6074a82f79b_ffbbe32019fd517ea21eab76e142aa7f store=[table=TestAcidGuarantees family=A region=ffbbe32019fd517ea21eab76e142aa7f] 2024-11-07T12:42:37,799 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241107847a43c2b01a47e1af75f6074a82f79b_ffbbe32019fd517ea21eab76e142aa7f, store=[table=TestAcidGuarantees family=A region=ffbbe32019fd517ea21eab76e142aa7f] 2024-11-07T12:42:37,799 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107847a43c2b01a47e1af75f6074a82f79b_ffbbe32019fd517ea21eab76e142aa7f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ffbbe32019fd517ea21eab76e142aa7f] 2024-11-07T12:42:37,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742238_1414 (size=12391) 2024-11-07T12:42:37,842 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-07T12:42:37,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:37,842 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing ffbbe32019fd517ea21eab76e142aa7f 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-07T12:42:37,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffbbe32019fd517ea21eab76e142aa7f, store=A 2024-11-07T12:42:37,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:37,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffbbe32019fd517ea21eab76e142aa7f, store=B 2024-11-07T12:42:37,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:37,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffbbe32019fd517ea21eab76e142aa7f, store=C 2024-11-07T12:42:37,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:37,843 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. as already flushing 2024-11-07T12:42:37,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:37,848 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/c36e10399d0047bcbf6f1e69fe05ab5b as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/c36e10399d0047bcbf6f1e69fe05ab5b 2024-11-07T12:42:37,857 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ffbbe32019fd517ea21eab76e142aa7f/B of ffbbe32019fd517ea21eab76e142aa7f into c36e10399d0047bcbf6f1e69fe05ab5b(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:42:37,857 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffbbe32019fd517ea21eab76e142aa7f: 2024-11-07T12:42:37,857 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., storeName=ffbbe32019fd517ea21eab76e142aa7f/B, priority=13, startTime=1730983357780; duration=0sec 2024-11-07T12:42:37,857 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:42:37,857 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffbbe32019fd517ea21eab76e142aa7f:B 2024-11-07T12:42:37,857 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:42:37,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742239_1415 (size=4469) 2024-11-07T12:42:37,861 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36291 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:42:37,861 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): ffbbe32019fd517ea21eab76e142aa7f/C is initiating minor compaction (all files) 2024-11-07T12:42:37,861 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffbbe32019fd517ea21eab76e142aa7f/C in TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:37,861 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/c74ce60b8291448ba153fbfb20c6fba4, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/465ff595d6e34831b549dddba14c03e1, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/c2d1ad17ea484d5bb8c61d4da470e282] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp, totalSize=35.4 K 2024-11-07T12:42:37,862 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting c74ce60b8291448ba153fbfb20c6fba4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1730983355329 2024-11-07T12:42:37,862 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 465ff595d6e34831b549dddba14c03e1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1730983355730 2024-11-07T12:42:37,862 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting c2d1ad17ea484d5bb8c61d4da470e282, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1730983356494 2024-11-07T12:42:37,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107014348ffd8864f4eb21707cb8522f470_ffbbe32019fd517ea21eab76e142aa7f is 50, key is test_row_0/A:col10/1730983357170/Put/seqid=0 2024-11-07T12:42:37,881 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffbbe32019fd517ea21eab76e142aa7f#C#compaction#357 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:42:37,881 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/6504ced6f2b74f66a318c8d9bbe617b8 is 50, key is test_row_0/C:col10/1730983357161/Put/seqid=0 2024-11-07T12:42:37,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742240_1416 (size=14794) 2024-11-07T12:42:37,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742241_1417 (size=12391) 2024-11-07T12:42:37,909 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1730983357865, max timestamp=1730983357905Num uniques 859; 2024-11-07T12:42:37,909 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-07T12:42:37,909 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1730983357865, max timestamp=1730983357905Num uniques 859; 2024-11-07T12:42:37,909 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-07T12:42:37,909 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1730983357865, max timestamp=1730983357905Num uniques 859; 2024-11-07T12:42:37,909 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-07T12:42:37,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:37,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55828 deadline: 1730983417948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:37,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:37,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:37,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:37,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:37,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55844 deadline: 1730983417949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:37,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:37,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55856 deadline: 1730983417950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:37,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:37,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:37,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:37,973 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:37,973 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:37,973 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:37,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:37,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55858 deadline: 1730983417957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:37,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:37,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55862 deadline: 1730983417958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:37,975 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:37,975 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:37,975 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:37,975 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:37,975 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:37,975 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:38,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-07T12:42:38,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:38,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55828 deadline: 1730983418071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:38,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:38,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:38,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:38,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:38,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55856 deadline: 1730983418082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:38,094 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:38,094 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:38,094 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:38,094 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:38,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55858 deadline: 1730983418082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:38,095 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:38,094 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:38,095 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:38,095 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:38,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55844 deadline: 1730983418082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:38,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:38,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:38,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:38,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55862 deadline: 1730983418082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:38,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:38,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:38,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:38,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:38,269 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffbbe32019fd517ea21eab76e142aa7f#A#compaction#354 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:42:38,269 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/4a4f40f77b934622ae165cdc19f6c6cf is 175, key is test_row_0/A:col10/1730983357161/Put/seqid=0 2024-11-07T12:42:38,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:38,298 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107014348ffd8864f4eb21707cb8522f470_ffbbe32019fd517ea21eab76e142aa7f to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107014348ffd8864f4eb21707cb8522f470_ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:38,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/e55dc0831f6b4de384dbccf65efd9e9e, store: [table=TestAcidGuarantees family=A region=ffbbe32019fd517ea21eab76e142aa7f] 2024-11-07T12:42:38,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/e55dc0831f6b4de384dbccf65efd9e9e is 175, key is test_row_0/A:col10/1730983357170/Put/seqid=0 2024-11-07T12:42:38,308 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/6504ced6f2b74f66a318c8d9bbe617b8 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/6504ced6f2b74f66a318c8d9bbe617b8 2024-11-07T12:42:38,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:38,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55828 deadline: 1730983418292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:38,312 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:38,312 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:38,312 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:38,313 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ffbbe32019fd517ea21eab76e142aa7f/C of ffbbe32019fd517ea21eab76e142aa7f into 6504ced6f2b74f66a318c8d9bbe617b8(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:42:38,313 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffbbe32019fd517ea21eab76e142aa7f: 2024-11-07T12:42:38,313 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., storeName=ffbbe32019fd517ea21eab76e142aa7f/C, priority=13, startTime=1730983357780; duration=0sec 2024-11-07T12:42:38,313 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:42:38,313 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffbbe32019fd517ea21eab76e142aa7f:C 2024-11-07T12:42:38,319 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:38,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55858 deadline: 1730983418302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:38,319 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:38,319 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:38,319 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:38,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:38,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55844 deadline: 1730983418303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:38,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:38,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:38,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:38,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:38,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55856 deadline: 1730983418303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:38,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:38,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742242_1418 (size=31345) 2024-11-07T12:42:38,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55862 deadline: 1730983418303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:38,322 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:38,322 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:38,322 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:38,323 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:38,323 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:38,323 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:38,333 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/4a4f40f77b934622ae165cdc19f6c6cf as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/4a4f40f77b934622ae165cdc19f6c6cf 2024-11-07T12:42:38,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742243_1419 (size=39749) 2024-11-07T12:42:38,339 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=156, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/e55dc0831f6b4de384dbccf65efd9e9e 2024-11-07T12:42:38,340 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ffbbe32019fd517ea21eab76e142aa7f/A of ffbbe32019fd517ea21eab76e142aa7f into 4a4f40f77b934622ae165cdc19f6c6cf(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:42:38,341 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffbbe32019fd517ea21eab76e142aa7f: 2024-11-07T12:42:38,341 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., storeName=ffbbe32019fd517ea21eab76e142aa7f/A, priority=13, startTime=1730983357780; duration=0sec 2024-11-07T12:42:38,341 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:42:38,341 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffbbe32019fd517ea21eab76e142aa7f:A 2024-11-07T12:42:38,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/1dec3da7e84a4599a06d15462113ba7c is 50, key is test_row_0/B:col10/1730983357170/Put/seqid=0 2024-11-07T12:42:38,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742244_1420 (size=12151) 2024-11-07T12:42:38,364 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/1dec3da7e84a4599a06d15462113ba7c 2024-11-07T12:42:38,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/cef3cf9278b440ce83042b61928f3b61 is 50, key is test_row_0/C:col10/1730983357170/Put/seqid=0 2024-11-07T12:42:38,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742245_1421 (size=12151) 2024-11-07T12:42:38,383 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/cef3cf9278b440ce83042b61928f3b61 2024-11-07T12:42:38,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/e55dc0831f6b4de384dbccf65efd9e9e as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/e55dc0831f6b4de384dbccf65efd9e9e 2024-11-07T12:42:38,406 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/e55dc0831f6b4de384dbccf65efd9e9e, entries=200, sequenceid=156, filesize=38.8 K 2024-11-07T12:42:38,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/1dec3da7e84a4599a06d15462113ba7c as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/1dec3da7e84a4599a06d15462113ba7c 2024-11-07T12:42:38,413 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/1dec3da7e84a4599a06d15462113ba7c, entries=150, sequenceid=156, filesize=11.9 K 2024-11-07T12:42:38,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/cef3cf9278b440ce83042b61928f3b61 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/cef3cf9278b440ce83042b61928f3b61 2024-11-07T12:42:38,418 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/cef3cf9278b440ce83042b61928f3b61, entries=150, sequenceid=156, filesize=11.9 K 2024-11-07T12:42:38,419 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for ffbbe32019fd517ea21eab76e142aa7f in 577ms, sequenceid=156, compaction requested=false 2024-11-07T12:42:38,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for ffbbe32019fd517ea21eab76e142aa7f: 2024-11-07T12:42:38,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:38,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-07T12:42:38,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-07T12:42:38,428 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-07T12:42:38,428 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0490 sec 2024-11-07T12:42:38,430 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 1.0560 sec 2024-11-07T12:42:38,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-07T12:42:38,513 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:42:38,514 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:42:38,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-07T12:42:38,516 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:42:38,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-07T12:42:38,520 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:42:38,520 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:42:38,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:38,629 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ffbbe32019fd517ea21eab76e142aa7f 3/3 column families, dataSize=147.60 KB heapSize=410.77 KB 2024-11-07T12:42:38,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffbbe32019fd517ea21eab76e142aa7f, store=A 2024-11-07T12:42:38,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:42:38,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffbbe32019fd517ea21eab76e142aa7f, store=B 2024-11-07T12:42:38,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:42:38,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffbbe32019fd517ea21eab76e142aa7f, store=C 2024-11-07T12:42:38,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:42:38,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-07T12:42:38,641 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411070d3bd09a54db427b8267cd4a1b93cd78_ffbbe32019fd517ea21eab76e142aa7f is 50, key is test_row_0/A:col10/1730983357947/Put/seqid=0 2024-11-07T12:42:38,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742246_1422 (size=14794) 2024-11-07T12:42:38,649 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:38,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:38,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55828 deadline: 1730983418641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:38,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:38,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:38,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:38,656 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411070d3bd09a54db427b8267cd4a1b93cd78_ffbbe32019fd517ea21eab76e142aa7f to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411070d3bd09a54db427b8267cd4a1b93cd78_ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:38,657 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/560b7b82979f40dba212c1c24b3b3895, store: [table=TestAcidGuarantees family=A region=ffbbe32019fd517ea21eab76e142aa7f] 2024-11-07T12:42:38,657 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/560b7b82979f40dba212c1c24b3b3895 is 175, key is test_row_0/A:col10/1730983357947/Put/seqid=0 2024-11-07T12:42:38,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742247_1423 (size=39749) 2024-11-07T12:42:38,662 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=195, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/560b7b82979f40dba212c1c24b3b3895 2024-11-07T12:42:38,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:38,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55844 deadline: 1730983418649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:38,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:38,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:38,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:38,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55856 deadline: 1730983418649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:38,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:38,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:38,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55858 deadline: 1730983418649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:38,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:38,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:38,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:38,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:38,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55862 deadline: 1730983418652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:38,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:38,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:38,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:38,670 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:38,670 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:38,670 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:38,673 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-07T12:42:38,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:38,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. as already flushing 2024-11-07T12:42:38,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:38,673 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:38,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:38,673 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/7c919ce66e0745dd9a3c683288bd86cf is 50, key is test_row_0/B:col10/1730983357947/Put/seqid=0 2024-11-07T12:42:38,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:38,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742248_1424 (size=12151) 2024-11-07T12:42:38,694 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/7c919ce66e0745dd9a3c683288bd86cf 2024-11-07T12:42:38,708 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/13f5e62eaefe43f1bd5c33ff327e0e09 is 50, key is test_row_0/C:col10/1730983357947/Put/seqid=0 2024-11-07T12:42:38,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742249_1425 (size=12151) 2024-11-07T12:42:38,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:38,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55828 deadline: 1730983418762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:38,768 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:38,768 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:38,768 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:38,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:38,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55844 deadline: 1730983418772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:38,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:38,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:38,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55858 deadline: 1730983418772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:38,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:38,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:38,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55856 deadline: 1730983418772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:38,785 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:38,785 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:38,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:38,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:38,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55862 deadline: 1730983418772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:38,786 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:38,786 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:38,786 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:38,787 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:38,787 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:38,787 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:38,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:38,830 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-07T12:42:38,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:38,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. as already flushing 2024-11-07T12:42:38,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:38,830 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:38,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:38,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:38,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-07T12:42:38,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:38,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55828 deadline: 1730983418972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:38,984 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:38,984 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:38,984 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:38,995 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-07T12:42:38,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:38,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. as already flushing 2024-11-07T12:42:38,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:38,996 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:38,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:38,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:39,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:39,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55844 deadline: 1730983418992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:39,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:39,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55856 deadline: 1730983418992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:39,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:39,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55858 deadline: 1730983418992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:39,001 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:39,001 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:39,001 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:39,002 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:39,002 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:39,002 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:39,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:39,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55862 deadline: 1730983418992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:39,004 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:39,004 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:39,004 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:39,004 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:39,004 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:39,004 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:39,114 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/13f5e62eaefe43f1bd5c33ff327e0e09 2024-11-07T12:42:39,119 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/560b7b82979f40dba212c1c24b3b3895 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/560b7b82979f40dba212c1c24b3b3895 2024-11-07T12:42:39,125 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/560b7b82979f40dba212c1c24b3b3895, entries=200, sequenceid=195, filesize=38.8 K 2024-11-07T12:42:39,125 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/7c919ce66e0745dd9a3c683288bd86cf as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/7c919ce66e0745dd9a3c683288bd86cf 2024-11-07T12:42:39,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,136 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/7c919ce66e0745dd9a3c683288bd86cf, entries=150, sequenceid=195, filesize=11.9 K 2024-11-07T12:42:39,136 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/13f5e62eaefe43f1bd5c33ff327e0e09 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/13f5e62eaefe43f1bd5c33ff327e0e09 2024-11-07T12:42:39,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,141 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/13f5e62eaefe43f1bd5c33ff327e0e09, entries=150, sequenceid=195, filesize=11.9 K 2024-11-07T12:42:39,141 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~410.72 KB/420576, currentSize=40.25 KB/41220 for ffbbe32019fd517ea21eab76e142aa7f in 512ms, sequenceid=195, compaction requested=true 2024-11-07T12:42:39,142 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ffbbe32019fd517ea21eab76e142aa7f: 2024-11-07T12:42:39,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffbbe32019fd517ea21eab76e142aa7f:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:42:39,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:42:39,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffbbe32019fd517ea21eab76e142aa7f:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:42:39,142 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:42:39,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:42:39,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffbbe32019fd517ea21eab76e142aa7f:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:42:39,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:42:39,142 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:42:39,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,143 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110843 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:42:39,143 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): ffbbe32019fd517ea21eab76e142aa7f/A is initiating minor compaction (all files) 2024-11-07T12:42:39,143 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffbbe32019fd517ea21eab76e142aa7f/A in TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:39,143 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:42:39,143 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/4a4f40f77b934622ae165cdc19f6c6cf, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/e55dc0831f6b4de384dbccf65efd9e9e, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/560b7b82979f40dba212c1c24b3b3895] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp, totalSize=108.2 K 2024-11-07T12:42:39,143 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): ffbbe32019fd517ea21eab76e142aa7f/B is initiating minor compaction (all files) 2024-11-07T12:42:39,143 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:39,143 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffbbe32019fd517ea21eab76e142aa7f/B in TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:39,143 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. files: [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/4a4f40f77b934622ae165cdc19f6c6cf, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/e55dc0831f6b4de384dbccf65efd9e9e, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/560b7b82979f40dba212c1c24b3b3895] 2024-11-07T12:42:39,143 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/c36e10399d0047bcbf6f1e69fe05ab5b, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/1dec3da7e84a4599a06d15462113ba7c, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/7c919ce66e0745dd9a3c683288bd86cf] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp, totalSize=35.8 K 2024-11-07T12:42:39,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,144 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a4f40f77b934622ae165cdc19f6c6cf, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1730983356494 2024-11-07T12:42:39,144 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting c36e10399d0047bcbf6f1e69fe05ab5b, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1730983356494 2024-11-07T12:42:39,144 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 1dec3da7e84a4599a06d15462113ba7c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1730983357170 2024-11-07T12:42:39,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,144 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting e55dc0831f6b4de384dbccf65efd9e9e, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1730983357170 2024-11-07T12:42:39,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-07T12:42:39,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,159 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c919ce66e0745dd9a3c683288bd86cf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1730983357947 2024-11-07T12:42:39,159 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 560b7b82979f40dba212c1c24b3b3895, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1730983357947 2024-11-07T12:42:39,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,161 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-07T12:42:39,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:39,161 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing ffbbe32019fd517ea21eab76e142aa7f 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-07T12:42:39,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffbbe32019fd517ea21eab76e142aa7f, store=A 2024-11-07T12:42:39,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:39,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffbbe32019fd517ea21eab76e142aa7f, store=B 2024-11-07T12:42:39,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:39,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffbbe32019fd517ea21eab76e142aa7f, store=C 2024-11-07T12:42:39,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:39,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,173 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ffbbe32019fd517ea21eab76e142aa7f] 2024-11-07T12:42:39,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,187 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffbbe32019fd517ea21eab76e142aa7f#B#compaction#364 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:42:39,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,187 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/e8ed0332b6614cff9a58548047e5e52f is 50, key is test_row_0/B:col10/1730983357947/Put/seqid=0 2024-11-07T12:42:39,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107f456ad5095924509b6f4773a75a597fd_ffbbe32019fd517ea21eab76e142aa7f is 50, key is test_row_1/A:col10/1730983358648/Put/seqid=0 2024-11-07T12:42:39,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,206 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241107f95c1d4bb0be4bd192f98d43b8c15b88_ffbbe32019fd517ea21eab76e142aa7f store=[table=TestAcidGuarantees family=A region=ffbbe32019fd517ea21eab76e142aa7f] 2024-11-07T12:42:39,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,208 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241107f95c1d4bb0be4bd192f98d43b8c15b88_ffbbe32019fd517ea21eab76e142aa7f, store=[table=TestAcidGuarantees family=A region=ffbbe32019fd517ea21eab76e142aa7f] 2024-11-07T12:42:39,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,208 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107f95c1d4bb0be4bd192f98d43b8c15b88_ffbbe32019fd517ea21eab76e142aa7f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ffbbe32019fd517ea21eab76e142aa7f] 2024-11-07T12:42:39,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742251_1427 (size=9814) 2024-11-07T12:42:39,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742250_1426 (size=12493) 2024-11-07T12:42:39,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,245 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107f456ad5095924509b6f4773a75a597fd_ffbbe32019fd517ea21eab76e142aa7f to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107f456ad5095924509b6f4773a75a597fd_ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:39,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/ccc7bc899cda46ada28346d351973b46, store: [table=TestAcidGuarantees family=A region=ffbbe32019fd517ea21eab76e142aa7f] 2024-11-07T12:42:39,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/ccc7bc899cda46ada28346d351973b46 is 175, key is test_row_1/A:col10/1730983358648/Put/seqid=0 2024-11-07T12:42:39,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,255 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/e8ed0332b6614cff9a58548047e5e52f as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/e8ed0332b6614cff9a58548047e5e52f 2024-11-07T12:42:39,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,263 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ffbbe32019fd517ea21eab76e142aa7f/B of ffbbe32019fd517ea21eab76e142aa7f into e8ed0332b6614cff9a58548047e5e52f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:42:39,263 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffbbe32019fd517ea21eab76e142aa7f: 2024-11-07T12:42:39,263 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., storeName=ffbbe32019fd517ea21eab76e142aa7f/B, priority=13, startTime=1730983359142; duration=0sec 2024-11-07T12:42:39,263 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:42:39,263 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffbbe32019fd517ea21eab76e142aa7f:B 2024-11-07T12:42:39,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,263 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:42:39,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,264 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:42:39,264 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): ffbbe32019fd517ea21eab76e142aa7f/C is initiating minor compaction (all files) 2024-11-07T12:42:39,264 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffbbe32019fd517ea21eab76e142aa7f/C in TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:39,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,264 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/6504ced6f2b74f66a318c8d9bbe617b8, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/cef3cf9278b440ce83042b61928f3b61, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/13f5e62eaefe43f1bd5c33ff327e0e09] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp, totalSize=35.8 K 2024-11-07T12:42:39,265 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 6504ced6f2b74f66a318c8d9bbe617b8, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1730983356494 2024-11-07T12:42:39,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,265 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting cef3cf9278b440ce83042b61928f3b61, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1730983357170 2024-11-07T12:42:39,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,266 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 13f5e62eaefe43f1bd5c33ff327e0e09, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1730983357947 2024-11-07T12:42:39,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742252_1428 (size=4469) 2024-11-07T12:42:39,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,287 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffbbe32019fd517ea21eab76e142aa7f#C#compaction#366 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:42:39,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,288 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/faf2b5e3d27e48268ead25bea4d06f91 is 50, key is test_row_0/C:col10/1730983357947/Put/seqid=0 2024-11-07T12:42:39,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742253_1429 (size=22461) 2024-11-07T12:42:39,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,289 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=204, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/ccc7bc899cda46ada28346d351973b46 2024-11-07T12:42:39,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/4be8d8e8d0aa451b88004950b85e4070 is 50, key is test_row_1/B:col10/1730983358648/Put/seqid=0 2024-11-07T12:42:39,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742254_1430 (size=12493) 2024-11-07T12:42:39,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,326 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/faf2b5e3d27e48268ead25bea4d06f91 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/faf2b5e3d27e48268ead25bea4d06f91 2024-11-07T12:42:39,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,332 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ffbbe32019fd517ea21eab76e142aa7f/C of ffbbe32019fd517ea21eab76e142aa7f into faf2b5e3d27e48268ead25bea4d06f91(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:42:39,332 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffbbe32019fd517ea21eab76e142aa7f: 2024-11-07T12:42:39,332 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., storeName=ffbbe32019fd517ea21eab76e142aa7f/C, priority=13, startTime=1730983359142; duration=0sec 2024-11-07T12:42:39,332 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:42:39,332 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffbbe32019fd517ea21eab76e142aa7f:C 2024-11-07T12:42:39,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:39,338 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. as already flushing 2024-11-07T12:42:39,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742255_1431 (size=9757) 2024-11-07T12:42:39,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,464 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1730983359430, max timestamp=1730983359461Num uniques 859; 2024-11-07T12:42:39,464 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-07T12:42:39,467 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1730983359430, max timestamp=1730983359461Num uniques 859; 2024-11-07T12:42:39,467 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1730983359430, max timestamp=1730983359461Num uniques 859; 2024-11-07T12:42:39,467 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-07T12:42:39,467 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-07T12:42:39,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:39,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55856 deadline: 1730983419511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:39,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:39,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:39,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:39,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:39,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55828 deadline: 1730983419512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:39,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:39,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55858 deadline: 1730983419513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:39,531 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:39,531 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:39,531 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:39,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:39,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55844 deadline: 1730983419514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:39,531 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:39,531 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:39,531 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:39,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:39,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:39,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:39,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:39,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55862 deadline: 1730983419527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:39,540 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:39,540 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:39,540 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:39,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:39,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55856 deadline: 1730983419631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:39,641 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:39,641 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:39,641 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:39,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:39,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55844 deadline: 1730983419642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:39,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:39,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:39,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:39,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:39,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55858 deadline: 1730983419642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:39,653 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:39,653 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:39,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:39,654 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:39,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55828 deadline: 1730983419642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:39,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:39,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:39,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:39,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:39,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55862 deadline: 1730983419651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:39,655 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:39,655 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:39,655 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:39,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-07T12:42:39,676 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffbbe32019fd517ea21eab76e142aa7f#A#compaction#363 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:42:39,677 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/59243edfc3594cf79d701d7ce848061f is 175, key is test_row_0/A:col10/1730983357947/Put/seqid=0 2024-11-07T12:42:39,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742256_1432 (size=31447) 2024-11-07T12:42:39,758 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/4be8d8e8d0aa451b88004950b85e4070 2024-11-07T12:42:39,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/1a81ab6b544a40c5b923e1e04549f763 is 50, key is test_row_1/C:col10/1730983358648/Put/seqid=0 2024-11-07T12:42:39,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742257_1433 (size=9757) 2024-11-07T12:42:39,773 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/1a81ab6b544a40c5b923e1e04549f763 2024-11-07T12:42:39,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/ccc7bc899cda46ada28346d351973b46 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/ccc7bc899cda46ada28346d351973b46 2024-11-07T12:42:39,787 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/ccc7bc899cda46ada28346d351973b46, entries=100, sequenceid=204, filesize=21.9 K 2024-11-07T12:42:39,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/4be8d8e8d0aa451b88004950b85e4070 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/4be8d8e8d0aa451b88004950b85e4070 2024-11-07T12:42:39,791 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/4be8d8e8d0aa451b88004950b85e4070, entries=100, sequenceid=204, filesize=9.5 K 2024-11-07T12:42:39,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/1a81ab6b544a40c5b923e1e04549f763 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/1a81ab6b544a40c5b923e1e04549f763 2024-11-07T12:42:39,806 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/1a81ab6b544a40c5b923e1e04549f763, entries=100, sequenceid=204, filesize=9.5 K 2024-11-07T12:42:39,807 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for ffbbe32019fd517ea21eab76e142aa7f in 646ms, sequenceid=204, compaction requested=false 2024-11-07T12:42:39,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for ffbbe32019fd517ea21eab76e142aa7f: 2024-11-07T12:42:39,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:39,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-07T12:42:39,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-07T12:42:39,814 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-07T12:42:39,814 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2890 sec 2024-11-07T12:42:39,816 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 1.3000 sec 2024-11-07T12:42:39,859 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ffbbe32019fd517ea21eab76e142aa7f 3/3 column families, dataSize=147.60 KB heapSize=420.84 KB 2024-11-07T12:42:39,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffbbe32019fd517ea21eab76e142aa7f, store=A 2024-11-07T12:42:39,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-07T12:42:39,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffbbe32019fd517ea21eab76e142aa7f, store=B 2024-11-07T12:42:39,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-07T12:42:39,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffbbe32019fd517ea21eab76e142aa7f, store=C 2024-11-07T12:42:39,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-07T12:42:39,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:39,877 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411073f5aeceb55ab435b9ecbc632a5d38379_ffbbe32019fd517ea21eab76e142aa7f is 50, key is test_row_0/A:col10/1730983359523/Put/seqid=0 2024-11-07T12:42:39,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742258_1434 (size=14794) 2024-11-07T12:42:39,893 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:39,896 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411073f5aeceb55ab435b9ecbc632a5d38379_ffbbe32019fd517ea21eab76e142aa7f to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411073f5aeceb55ab435b9ecbc632a5d38379_ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:39,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:39,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55862 deadline: 1730983419880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:39,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:39,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55844 deadline: 1730983419880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:39,897 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:39,897 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:39,897 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:39,897 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/76c41d13c81f4961af8f6f0465b2dba1, store: [table=TestAcidGuarantees family=A region=ffbbe32019fd517ea21eab76e142aa7f] 2024-11-07T12:42:39,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:39,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:39,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:39,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:39,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55858 deadline: 1730983419880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:39,898 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/76c41d13c81f4961af8f6f0465b2dba1 is 175, key is test_row_0/A:col10/1730983359523/Put/seqid=0 2024-11-07T12:42:39,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:39,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:39,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:39,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:39,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55856 deadline: 1730983419882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:39,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:39,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:39,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:39,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:39,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55828 deadline: 1730983419883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:39,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:39,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:39,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:39,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742259_1435 (size=39749) 2024-11-07T12:42:39,913 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=244, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/76c41d13c81f4961af8f6f0465b2dba1 2024-11-07T12:42:39,943 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/a9e85e4fe3e44fc29aa8dd416e4418ed is 50, key is test_row_0/B:col10/1730983359523/Put/seqid=0 2024-11-07T12:42:39,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742260_1436 (size=12151) 2024-11-07T12:42:39,954 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/a9e85e4fe3e44fc29aa8dd416e4418ed 2024-11-07T12:42:39,975 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/16f8eacb8fdb4d31982ca6aa71a8b6ff is 50, key is test_row_0/C:col10/1730983359523/Put/seqid=0 2024-11-07T12:42:39,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742261_1437 (size=12151) 2024-11-07T12:42:39,991 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/16f8eacb8fdb4d31982ca6aa71a8b6ff 2024-11-07T12:42:40,002 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/76c41d13c81f4961af8f6f0465b2dba1 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/76c41d13c81f4961af8f6f0465b2dba1 2024-11-07T12:42:40,005 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/76c41d13c81f4961af8f6f0465b2dba1, entries=200, sequenceid=244, filesize=38.8 K 2024-11-07T12:42:40,006 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/a9e85e4fe3e44fc29aa8dd416e4418ed as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/a9e85e4fe3e44fc29aa8dd416e4418ed 2024-11-07T12:42:40,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:40,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55844 deadline: 1730983420001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:40,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:40,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:40,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:40,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:40,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55856 deadline: 1730983420002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:40,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:40,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:40,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:40,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55858 deadline: 1730983420002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:40,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:40,015 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:40,015 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:40,015 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:40,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:40,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55862 deadline: 1730983420002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:40,016 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:40,016 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:40,016 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:40,019 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/a9e85e4fe3e44fc29aa8dd416e4418ed, entries=150, sequenceid=244, filesize=11.9 K 2024-11-07T12:42:40,020 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/16f8eacb8fdb4d31982ca6aa71a8b6ff as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/16f8eacb8fdb4d31982ca6aa71a8b6ff 2024-11-07T12:42:40,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,026 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/16f8eacb8fdb4d31982ca6aa71a8b6ff, entries=150, sequenceid=244, filesize=11.9 K 2024-11-07T12:42:40,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,028 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~420.80 KB/430896, currentSize=33.54 KB/34350 for ffbbe32019fd517ea21eab76e142aa7f in 169ms, sequenceid=244, compaction requested=true 2024-11-07T12:42:40,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ffbbe32019fd517ea21eab76e142aa7f: 2024-11-07T12:42:40,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffbbe32019fd517ea21eab76e142aa7f:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:42:40,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:42:40,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffbbe32019fd517ea21eab76e142aa7f:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:42:40,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:42:40,028 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 3 compacting, 2 eligible, 16 blocking 2024-11-07T12:42:40,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffbbe32019fd517ea21eab76e142aa7f:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:42:40,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-07T12:42:40,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,029 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-07T12:42:40,029 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-07T12:42:40,030 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. because compaction request was cancelled 2024-11-07T12:42:40,030 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffbbe32019fd517ea21eab76e142aa7f:A 2024-11-07T12:42:40,030 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:42:40,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,031 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:42:40,031 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): ffbbe32019fd517ea21eab76e142aa7f/B is initiating minor compaction (all files) 2024-11-07T12:42:40,031 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffbbe32019fd517ea21eab76e142aa7f/B in TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:40,031 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/e8ed0332b6614cff9a58548047e5e52f, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/4be8d8e8d0aa451b88004950b85e4070, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/a9e85e4fe3e44fc29aa8dd416e4418ed] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp, totalSize=33.6 K 2024-11-07T12:42:40,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,031 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting e8ed0332b6614cff9a58548047e5e52f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1730983357947 2024-11-07T12:42:40,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,032 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 4be8d8e8d0aa451b88004950b85e4070, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1730983358646 2024-11-07T12:42:40,032 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting a9e85e4fe3e44fc29aa8dd416e4418ed, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1730983359512 2024-11-07T12:42:40,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,052 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffbbe32019fd517ea21eab76e142aa7f#B#compaction#372 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:42:40,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,052 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/540e32cb1b1a41b6a74340380f431840 is 50, key is test_row_0/B:col10/1730983359523/Put/seqid=0 2024-11-07T12:42:40,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,092 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/59243edfc3594cf79d701d7ce848061f as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/59243edfc3594cf79d701d7ce848061f 2024-11-07T12:42:40,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742262_1438 (size=12595) 2024-11-07T12:42:40,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,102 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ffbbe32019fd517ea21eab76e142aa7f/A of ffbbe32019fd517ea21eab76e142aa7f into 59243edfc3594cf79d701d7ce848061f(size=30.7 K), total size for store is 91.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:42:40,102 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffbbe32019fd517ea21eab76e142aa7f: 2024-11-07T12:42:40,102 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., storeName=ffbbe32019fd517ea21eab76e142aa7f/A, priority=13, startTime=1730983359142; duration=0sec 2024-11-07T12:42:40,102 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:42:40,102 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffbbe32019fd517ea21eab76e142aa7f:A 2024-11-07T12:42:40,102 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:42:40,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,103 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:42:40,103 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): ffbbe32019fd517ea21eab76e142aa7f/C is initiating minor compaction (all files) 2024-11-07T12:42:40,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,103 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffbbe32019fd517ea21eab76e142aa7f/C in TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:40,103 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/faf2b5e3d27e48268ead25bea4d06f91, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/1a81ab6b544a40c5b923e1e04549f763, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/16f8eacb8fdb4d31982ca6aa71a8b6ff] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp, totalSize=33.6 K 2024-11-07T12:42:40,104 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting faf2b5e3d27e48268ead25bea4d06f91, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1730983357947 2024-11-07T12:42:40,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,104 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a81ab6b544a40c5b923e1e04549f763, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1730983358646 2024-11-07T12:42:40,105 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16f8eacb8fdb4d31982ca6aa71a8b6ff, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1730983359512 2024-11-07T12:42:40,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,108 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/540e32cb1b1a41b6a74340380f431840 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/540e32cb1b1a41b6a74340380f431840 2024-11-07T12:42:40,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,113 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ffbbe32019fd517ea21eab76e142aa7f/B of ffbbe32019fd517ea21eab76e142aa7f into 540e32cb1b1a41b6a74340380f431840(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:42:40,113 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffbbe32019fd517ea21eab76e142aa7f: 2024-11-07T12:42:40,114 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., storeName=ffbbe32019fd517ea21eab76e142aa7f/B, priority=13, startTime=1730983360028; duration=0sec 2024-11-07T12:42:40,114 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:42:40,114 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffbbe32019fd517ea21eab76e142aa7f:B 2024-11-07T12:42:40,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,123 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffbbe32019fd517ea21eab76e142aa7f#C#compaction#373 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:42:40,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,124 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/7648d58a095140fa8ac33172dabcee71 is 50, key is test_row_0/C:col10/1730983359523/Put/seqid=0 2024-11-07T12:42:40,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,144 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ffbbe32019fd517ea21eab76e142aa7f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-07T12:42:40,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffbbe32019fd517ea21eab76e142aa7f, store=A 2024-11-07T12:42:40,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:40,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffbbe32019fd517ea21eab76e142aa7f, store=B 2024-11-07T12:42:40,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:40,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffbbe32019fd517ea21eab76e142aa7f, store=C 2024-11-07T12:42:40,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:40,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:40,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742263_1439 (size=12595) 2024-11-07T12:42:40,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,167 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/7648d58a095140fa8ac33172dabcee71 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/7648d58a095140fa8ac33172dabcee71 2024-11-07T12:42:40,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,172 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ffbbe32019fd517ea21eab76e142aa7f/C of ffbbe32019fd517ea21eab76e142aa7f into 7648d58a095140fa8ac33172dabcee71(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:42:40,172 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffbbe32019fd517ea21eab76e142aa7f: 2024-11-07T12:42:40,172 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., storeName=ffbbe32019fd517ea21eab76e142aa7f/C, priority=13, startTime=1730983360028; duration=0sec 2024-11-07T12:42:40,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,172 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:42:40,172 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffbbe32019fd517ea21eab76e142aa7f:C 2024-11-07T12:42:40,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,174 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411070eed8b74546643bd821237a9909f3ca1_ffbbe32019fd517ea21eab76e142aa7f is 50, key is test_row_0/A:col10/1730983360074/Put/seqid=0 2024-11-07T12:42:40,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742264_1440 (size=12304) 2024-11-07T12:42:40,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,223 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,226 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411070eed8b74546643bd821237a9909f3ca1_ffbbe32019fd517ea21eab76e142aa7f to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411070eed8b74546643bd821237a9909f3ca1_ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:40,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,227 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/ccabddd91a804b9f861640b5380bafed, store: [table=TestAcidGuarantees family=A region=ffbbe32019fd517ea21eab76e142aa7f] 2024-11-07T12:42:40,228 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/ccabddd91a804b9f861640b5380bafed is 175, key is test_row_0/A:col10/1730983360074/Put/seqid=0 2024-11-07T12:42:40,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:40,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742265_1441 (size=31101) 2024-11-07T12:42:40,261 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=256, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/ccabddd91a804b9f861640b5380bafed 2024-11-07T12:42:40,291 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/4aa685ff56f24ff6ad79594995c1d75b is 50, key is test_row_0/B:col10/1730983360074/Put/seqid=0 2024-11-07T12:42:40,318 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1730983360293, max timestamp=1730983360314Num uniques 859; 2024-11-07T12:42:40,318 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-07T12:42:40,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742266_1442 (size=9757) 2024-11-07T12:42:40,337 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/4aa685ff56f24ff6ad79594995c1d75b 2024-11-07T12:42:40,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:40,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55862 deadline: 1730983420323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:40,345 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:40,345 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:40,345 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:40,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:40,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55856 deadline: 1730983420325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:40,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:40,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:40,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:40,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:40,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55828 deadline: 1730983420325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:40,346 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:40,346 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:40,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:40,359 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/96ebafeecff1423888688a233f19b19a is 50, key is test_row_0/C:col10/1730983360074/Put/seqid=0 2024-11-07T12:42:40,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:40,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55844 deadline: 1730983420339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:40,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:40,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:40,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:40,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:40,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55858 deadline: 1730983420341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:40,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:40,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:40,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:40,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742267_1443 (size=9757) 2024-11-07T12:42:40,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:40,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55862 deadline: 1730983420452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:40,462 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:40,462 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:40,462 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:40,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:40,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55856 deadline: 1730983420452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:40,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:40,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:40,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:40,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:40,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55828 deadline: 1730983420453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:40,463 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:40,463 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:40,463 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:40,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:40,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55858 deadline: 1730983420472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:40,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:40,483 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:40,483 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:40,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55844 deadline: 1730983420473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:40,483 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:40,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:40,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:40,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:40,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-07T12:42:40,672 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:42:40,672 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:42:40,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-07T12:42:40,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-07T12:42:40,674 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:42:40,675 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:42:40,675 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:42:40,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:40,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55862 deadline: 1730983420671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:40,686 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:40,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55856 deadline: 1730983420671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:40,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:40,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:40,686 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:40,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:40,686 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:40,686 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:40,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55828 deadline: 1730983420672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:40,686 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:40,687 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:40,687 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:40,687 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:40,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:40,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55858 deadline: 1730983420692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:40,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:40,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55844 deadline: 1730983420697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:40,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:40,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:40,704 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:40,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:40,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:40,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:40,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-07T12:42:40,818 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/96ebafeecff1423888688a233f19b19a 2024-11-07T12:42:40,827 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/ccabddd91a804b9f861640b5380bafed as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/ccabddd91a804b9f861640b5380bafed 2024-11-07T12:42:40,828 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-07T12:42:40,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:40,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. as already flushing 2024-11-07T12:42:40,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:40,829 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:40,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:40,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:40,834 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/ccabddd91a804b9f861640b5380bafed, entries=150, sequenceid=256, filesize=30.4 K 2024-11-07T12:42:40,835 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/4aa685ff56f24ff6ad79594995c1d75b as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/4aa685ff56f24ff6ad79594995c1d75b 2024-11-07T12:42:40,839 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/4aa685ff56f24ff6ad79594995c1d75b, entries=100, sequenceid=256, filesize=9.5 K 2024-11-07T12:42:40,841 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/96ebafeecff1423888688a233f19b19a as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/96ebafeecff1423888688a233f19b19a 2024-11-07T12:42:40,847 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/96ebafeecff1423888688a233f19b19a, entries=100, sequenceid=256, filesize=9.5 K 2024-11-07T12:42:40,849 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=129.71 KB/132820 for ffbbe32019fd517ea21eab76e142aa7f in 704ms, sequenceid=256, compaction requested=true 2024-11-07T12:42:40,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ffbbe32019fd517ea21eab76e142aa7f: 2024-11-07T12:42:40,849 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:42:40,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffbbe32019fd517ea21eab76e142aa7f:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:42:40,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:42:40,850 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-07T12:42:40,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffbbe32019fd517ea21eab76e142aa7f:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:42:40,850 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 124758 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:42:40,850 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): ffbbe32019fd517ea21eab76e142aa7f/A is initiating minor compaction (all files) 2024-11-07T12:42:40,850 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffbbe32019fd517ea21eab76e142aa7f/A in TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:40,850 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/59243edfc3594cf79d701d7ce848061f, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/ccc7bc899cda46ada28346d351973b46, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/76c41d13c81f4961af8f6f0465b2dba1, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/ccabddd91a804b9f861640b5380bafed] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp, totalSize=121.8 K 2024-11-07T12:42:40,850 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:40,850 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. files: [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/59243edfc3594cf79d701d7ce848061f, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/ccc7bc899cda46ada28346d351973b46, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/76c41d13c81f4961af8f6f0465b2dba1, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/ccabddd91a804b9f861640b5380bafed] 2024-11-07T12:42:40,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:42:40,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffbbe32019fd517ea21eab76e142aa7f:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:42:40,851 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-07T12:42:40,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:42:40,851 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-07T12:42:40,851 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. because compaction request was cancelled 2024-11-07T12:42:40,851 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffbbe32019fd517ea21eab76e142aa7f:B 2024-11-07T12:42:40,851 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-07T12:42:40,852 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59243edfc3594cf79d701d7ce848061f, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1730983357947 2024-11-07T12:42:40,852 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting ccc7bc899cda46ada28346d351973b46, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1730983358646 2024-11-07T12:42:40,852 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-07T12:42:40,852 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-07T12:42:40,852 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. because compaction request was cancelled 2024-11-07T12:42:40,852 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffbbe32019fd517ea21eab76e142aa7f:C 2024-11-07T12:42:40,852 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76c41d13c81f4961af8f6f0465b2dba1, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1730983359473 2024-11-07T12:42:40,853 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting ccabddd91a804b9f861640b5380bafed, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1730983359879 2024-11-07T12:42:40,873 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ffbbe32019fd517ea21eab76e142aa7f] 2024-11-07T12:42:40,895 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024110754445019e97d4a8bae7c0e931f5bbe3d_ffbbe32019fd517ea21eab76e142aa7f store=[table=TestAcidGuarantees family=A region=ffbbe32019fd517ea21eab76e142aa7f] 2024-11-07T12:42:40,898 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024110754445019e97d4a8bae7c0e931f5bbe3d_ffbbe32019fd517ea21eab76e142aa7f, store=[table=TestAcidGuarantees family=A region=ffbbe32019fd517ea21eab76e142aa7f] 2024-11-07T12:42:40,898 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110754445019e97d4a8bae7c0e931f5bbe3d_ffbbe32019fd517ea21eab76e142aa7f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ffbbe32019fd517ea21eab76e142aa7f] 2024-11-07T12:42:40,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742268_1444 (size=4469) 2024-11-07T12:42:40,930 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffbbe32019fd517ea21eab76e142aa7f#A#compaction#377 average throughput is 0.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:42:40,930 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/fa9e7da4d0c14ebf92cfcb6b8936146f is 175, key is test_row_0/A:col10/1730983360074/Put/seqid=0 2024-11-07T12:42:40,982 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-07T12:42:40,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:40,982 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing ffbbe32019fd517ea21eab76e142aa7f 3/3 column families, dataSize=129.71 KB heapSize=381.64 KB 2024-11-07T12:42:40,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffbbe32019fd517ea21eab76e142aa7f, store=A 2024-11-07T12:42:40,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:42:40,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffbbe32019fd517ea21eab76e142aa7f, store=B 2024-11-07T12:42:40,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:42:40,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffbbe32019fd517ea21eab76e142aa7f, store=C 2024-11-07T12:42:40,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:42:40,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-07T12:42:40,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742269_1445 (size=31690) 2024-11-07T12:42:41,007 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. as already flushing 2024-11-07T12:42:41,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:41,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110703969be376c94830bd5b7ccec3cbcef4_ffbbe32019fd517ea21eab76e142aa7f is 50, key is test_row_0/A:col10/1730983360334/Put/seqid=0 2024-11-07T12:42:41,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:41,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55844 deadline: 1730983421017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:41,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:41,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55856 deadline: 1730983421021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:41,051 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:41,051 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:41,051 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:41,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:41,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:41,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:41,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:41,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55828 deadline: 1730983421046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:41,073 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:41,073 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:41,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:41,073 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:41,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55862 deadline: 1730983421049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:41,073 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:41,073 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:41,073 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:41,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:41,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55858 deadline: 1730983421053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:41,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:41,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:41,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:41,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742270_1446 (size=12454) 2024-11-07T12:42:41,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:41,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55856 deadline: 1730983421161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:41,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:41,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:41,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:41,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:41,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55828 deadline: 1730983421182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:41,207 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:41,207 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:41,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:41,207 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:41,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55858 deadline: 1730983421182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:41,207 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:41,207 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:41,207 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:41,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:41,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55862 deadline: 1730983421182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:41,208 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:41,208 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:41,208 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:41,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-07T12:42:41,399 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/fa9e7da4d0c14ebf92cfcb6b8936146f as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/fa9e7da4d0c14ebf92cfcb6b8936146f 2024-11-07T12:42:41,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:41,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55856 deadline: 1730983421392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:41,400 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:41,400 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:41,400 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:41,409 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ffbbe32019fd517ea21eab76e142aa7f/A of ffbbe32019fd517ea21eab76e142aa7f into fa9e7da4d0c14ebf92cfcb6b8936146f(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:42:41,409 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffbbe32019fd517ea21eab76e142aa7f: 2024-11-07T12:42:41,409 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., storeName=ffbbe32019fd517ea21eab76e142aa7f/A, priority=12, startTime=1730983360849; duration=0sec 2024-11-07T12:42:41,409 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:42:41,409 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffbbe32019fd517ea21eab76e142aa7f:A 2024-11-07T12:42:41,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:41,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55828 deadline: 1730983421412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:41,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:41,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55858 deadline: 1730983421412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:41,425 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:41,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:41,425 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:41,425 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:41,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55862 deadline: 1730983421412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:41,425 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:41,425 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:41,425 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:41,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:41,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:41,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:41,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:41,491 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110703969be376c94830bd5b7ccec3cbcef4_ffbbe32019fd517ea21eab76e142aa7f to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110703969be376c94830bd5b7ccec3cbcef4_ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:41,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/39de740b13244d4ebd4ca237d90fec5a, store: [table=TestAcidGuarantees family=A region=ffbbe32019fd517ea21eab76e142aa7f] 2024-11-07T12:42:41,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/39de740b13244d4ebd4ca237d90fec5a is 175, key is test_row_0/A:col10/1730983360334/Put/seqid=0 2024-11-07T12:42:41,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742271_1447 (size=31255) 2024-11-07T12:42:41,508 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=284, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/39de740b13244d4ebd4ca237d90fec5a 2024-11-07T12:42:41,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/a3531242048d4e3fbfd9663ae2875e0c is 50, key is test_row_0/B:col10/1730983360334/Put/seqid=0 2024-11-07T12:42:41,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742272_1448 (size=12301) 2024-11-07T12:42:41,529 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/a3531242048d4e3fbfd9663ae2875e0c 2024-11-07T12:42:41,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/8da116fb23314cdebd8461f50d83a03a is 50, key is test_row_0/C:col10/1730983360334/Put/seqid=0 2024-11-07T12:42:41,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742273_1449 (size=12301) 2024-11-07T12:42:41,551 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/8da116fb23314cdebd8461f50d83a03a 2024-11-07T12:42:41,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/39de740b13244d4ebd4ca237d90fec5a as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/39de740b13244d4ebd4ca237d90fec5a 2024-11-07T12:42:41,559 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/39de740b13244d4ebd4ca237d90fec5a, entries=150, sequenceid=284, filesize=30.5 K 2024-11-07T12:42:41,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/a3531242048d4e3fbfd9663ae2875e0c as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/a3531242048d4e3fbfd9663ae2875e0c 2024-11-07T12:42:41,565 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/a3531242048d4e3fbfd9663ae2875e0c, entries=150, sequenceid=284, filesize=12.0 K 2024-11-07T12:42:41,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/8da116fb23314cdebd8461f50d83a03a as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/8da116fb23314cdebd8461f50d83a03a 2024-11-07T12:42:41,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:41,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55844 deadline: 1730983421561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:41,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:41,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:41,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:41,574 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/8da116fb23314cdebd8461f50d83a03a, entries=150, sequenceid=284, filesize=12.0 K 2024-11-07T12:42:41,575 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~129.71 KB/132820, heapSize ~381.59 KB/390752, currentSize=46.96 KB/48090 for ffbbe32019fd517ea21eab76e142aa7f in 593ms, sequenceid=284, compaction requested=true 2024-11-07T12:42:41,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for ffbbe32019fd517ea21eab76e142aa7f: 2024-11-07T12:42:41,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:41,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-07T12:42:41,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-07T12:42:41,582 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-07T12:42:41,582 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 905 msec 2024-11-07T12:42:41,584 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 910 msec 2024-11-07T12:42:41,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:41,720 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ffbbe32019fd517ea21eab76e142aa7f 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-07T12:42:41,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffbbe32019fd517ea21eab76e142aa7f, store=A 2024-11-07T12:42:41,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:41,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffbbe32019fd517ea21eab76e142aa7f, store=B 2024-11-07T12:42:41,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:41,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffbbe32019fd517ea21eab76e142aa7f, store=C 2024-11-07T12:42:41,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:41,747 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411078a22a9ff67f94c2d80cf2e5e7112e6c4_ffbbe32019fd517ea21eab76e142aa7f is 50, key is test_row_0/A:col10/1730983361718/Put/seqid=0 2024-11-07T12:42:41,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742274_1450 (size=14994) 2024-11-07T12:42:41,795 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1730983361763, max timestamp=1730983361792Num uniques 859; 2024-11-07T12:42:41,795 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-07T12:42:41,795 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1730983361763, max timestamp=1730983361792Num uniques 859; 2024-11-07T12:42:41,795 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-07T12:42:41,795 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1730983361763, max timestamp=1730983361792Num uniques 859; 2024-11-07T12:42:41,796 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-07T12:42:41,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-07T12:42:41,811 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:42:41,812 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:42:41,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-07T12:42:41,814 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:42:41,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-07T12:42:41,816 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:42:41,817 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:42:41,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:41,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55862 deadline: 1730983421836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:41,861 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:41,861 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:41,861 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:41,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:41,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55828 deadline: 1730983421841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:41,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:41,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:41,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:41,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:41,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55856 deadline: 1730983421851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:41,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:41,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:41,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:41,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:41,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55858 deadline: 1730983421860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:41,885 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:41,885 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:41,885 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:41,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-07T12:42:41,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-07T12:42:41,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:41,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. as already flushing 2024-11-07T12:42:41,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:41,969 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:41,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:41,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:41,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:41,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55828 deadline: 1730983421972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:41,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:41,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55862 deadline: 1730983421972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:41,980 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:41,980 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:41,980 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:41,981 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:41,981 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:41,981 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:42,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:42,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55856 deadline: 1730983421982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:42,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:42,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55858 deadline: 1730983421991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:42,001 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:42,001 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:42,001 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:42,001 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:42,001 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:42,001 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:42,122 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-07T12:42:42,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:42,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. as already flushing 2024-11-07T12:42:42,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:42,123 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:42,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:42,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:42,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-07T12:42:42,162 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:42,166 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411078a22a9ff67f94c2d80cf2e5e7112e6c4_ffbbe32019fd517ea21eab76e142aa7f to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411078a22a9ff67f94c2d80cf2e5e7112e6c4_ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:42,167 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/97cf8893e9674643930515cd53522401, store: [table=TestAcidGuarantees family=A region=ffbbe32019fd517ea21eab76e142aa7f] 2024-11-07T12:42:42,168 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/97cf8893e9674643930515cd53522401 is 175, key is test_row_0/A:col10/1730983361718/Put/seqid=0 2024-11-07T12:42:42,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742275_1451 (size=39949) 2024-11-07T12:42:42,185 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=296, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/97cf8893e9674643930515cd53522401 2024-11-07T12:42:42,194 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/f6cd9e5795954b1da4597f3d769e0468 is 50, key is test_row_0/B:col10/1730983361718/Put/seqid=0 2024-11-07T12:42:42,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:42,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55828 deadline: 1730983422192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:42,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:42,202 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:42,202 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:42,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55862 deadline: 1730983422192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:42,202 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:42,204 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:42,205 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:42,205 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:42,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:42,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55858 deadline: 1730983422211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:42,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:42,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:42,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:42,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:42,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55856 deadline: 1730983422211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:42,220 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:42,220 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:42,220 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:42,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742276_1452 (size=12301) 2024-11-07T12:42:42,232 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/f6cd9e5795954b1da4597f3d769e0468 2024-11-07T12:42:42,243 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/1d106a1eacd74c869ae48dab987b4f81 is 50, key is test_row_0/C:col10/1730983361718/Put/seqid=0 2024-11-07T12:42:42,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742277_1453 (size=12301) 2024-11-07T12:42:42,266 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/1d106a1eacd74c869ae48dab987b4f81 2024-11-07T12:42:42,271 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/97cf8893e9674643930515cd53522401 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/97cf8893e9674643930515cd53522401 2024-11-07T12:42:42,275 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/97cf8893e9674643930515cd53522401, entries=200, sequenceid=296, filesize=39.0 K 2024-11-07T12:42:42,276 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-07T12:42:42,276 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/f6cd9e5795954b1da4597f3d769e0468 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/f6cd9e5795954b1da4597f3d769e0468 2024-11-07T12:42:42,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:42,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. as already flushing 2024-11-07T12:42:42,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:42,277 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:42,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:42,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:42,286 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/f6cd9e5795954b1da4597f3d769e0468, entries=150, sequenceid=296, filesize=12.0 K 2024-11-07T12:42:42,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/1d106a1eacd74c869ae48dab987b4f81 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/1d106a1eacd74c869ae48dab987b4f81 2024-11-07T12:42:42,296 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/1d106a1eacd74c869ae48dab987b4f81, entries=150, sequenceid=296, filesize=12.0 K 2024-11-07T12:42:42,297 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for ffbbe32019fd517ea21eab76e142aa7f in 577ms, sequenceid=296, compaction requested=true 2024-11-07T12:42:42,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ffbbe32019fd517ea21eab76e142aa7f: 2024-11-07T12:42:42,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffbbe32019fd517ea21eab76e142aa7f:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:42:42,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:42:42,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffbbe32019fd517ea21eab76e142aa7f:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:42:42,297 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:42:42,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:42:42,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffbbe32019fd517ea21eab76e142aa7f:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:42:42,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:42:42,297 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:42:42,299 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102894 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:42:42,299 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): ffbbe32019fd517ea21eab76e142aa7f/A is initiating minor compaction (all files) 2024-11-07T12:42:42,299 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffbbe32019fd517ea21eab76e142aa7f/A in TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:42,300 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/fa9e7da4d0c14ebf92cfcb6b8936146f, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/39de740b13244d4ebd4ca237d90fec5a, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/97cf8893e9674643930515cd53522401] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp, totalSize=100.5 K 2024-11-07T12:42:42,300 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:42,300 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. files: [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/fa9e7da4d0c14ebf92cfcb6b8936146f, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/39de740b13244d4ebd4ca237d90fec5a, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/97cf8893e9674643930515cd53522401] 2024-11-07T12:42:42,300 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:42:42,300 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): ffbbe32019fd517ea21eab76e142aa7f/B is initiating minor compaction (all files) 2024-11-07T12:42:42,300 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffbbe32019fd517ea21eab76e142aa7f/B in TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:42,300 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/540e32cb1b1a41b6a74340380f431840, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/4aa685ff56f24ff6ad79594995c1d75b, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/a3531242048d4e3fbfd9663ae2875e0c, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/f6cd9e5795954b1da4597f3d769e0468] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp, totalSize=45.9 K 2024-11-07T12:42:42,301 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa9e7da4d0c14ebf92cfcb6b8936146f, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1730983359512 2024-11-07T12:42:42,302 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 540e32cb1b1a41b6a74340380f431840, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1730983359512 2024-11-07T12:42:42,302 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 39de740b13244d4ebd4ca237d90fec5a, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1730983360321 2024-11-07T12:42:42,303 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 97cf8893e9674643930515cd53522401, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1730983361011 2024-11-07T12:42:42,303 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 4aa685ff56f24ff6ad79594995c1d75b, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1730983360074 2024-11-07T12:42:42,303 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting a3531242048d4e3fbfd9663ae2875e0c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1730983360321 2024-11-07T12:42:42,304 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting f6cd9e5795954b1da4597f3d769e0468, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1730983361035 2024-11-07T12:42:42,312 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ffbbe32019fd517ea21eab76e142aa7f] 2024-11-07T12:42:42,325 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241107789c70a9944342fc91a0690cc444ef4f_ffbbe32019fd517ea21eab76e142aa7f store=[table=TestAcidGuarantees family=A region=ffbbe32019fd517ea21eab76e142aa7f] 2024-11-07T12:42:42,327 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241107789c70a9944342fc91a0690cc444ef4f_ffbbe32019fd517ea21eab76e142aa7f, store=[table=TestAcidGuarantees family=A region=ffbbe32019fd517ea21eab76e142aa7f] 2024-11-07T12:42:42,327 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107789c70a9944342fc91a0690cc444ef4f_ffbbe32019fd517ea21eab76e142aa7f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ffbbe32019fd517ea21eab76e142aa7f] 2024-11-07T12:42:42,339 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffbbe32019fd517ea21eab76e142aa7f#B#compaction#385 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:42:42,340 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/c3691d6c0606412d8710ee14052e779e is 50, key is test_row_0/B:col10/1730983361718/Put/seqid=0 2024-11-07T12:42:42,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742278_1454 (size=4469) 2024-11-07T12:42:42,361 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffbbe32019fd517ea21eab76e142aa7f#A#compaction#384 average throughput is 0.50 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:42:42,362 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/553fb7f407c74aff9cbc4445fc96f9e2 is 175, key is test_row_0/A:col10/1730983361718/Put/seqid=0 2024-11-07T12:42:42,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742279_1455 (size=12881) 2024-11-07T12:42:42,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742280_1456 (size=31835) 2024-11-07T12:42:42,407 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/553fb7f407c74aff9cbc4445fc96f9e2 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/553fb7f407c74aff9cbc4445fc96f9e2 2024-11-07T12:42:42,417 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ffbbe32019fd517ea21eab76e142aa7f/A of ffbbe32019fd517ea21eab76e142aa7f into 553fb7f407c74aff9cbc4445fc96f9e2(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:42:42,417 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffbbe32019fd517ea21eab76e142aa7f: 2024-11-07T12:42:42,417 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., storeName=ffbbe32019fd517ea21eab76e142aa7f/A, priority=13, startTime=1730983362297; duration=0sec 2024-11-07T12:42:42,417 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:42:42,417 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffbbe32019fd517ea21eab76e142aa7f:A 2024-11-07T12:42:42,417 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:42:42,421 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:42:42,422 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): ffbbe32019fd517ea21eab76e142aa7f/C is initiating minor compaction (all files) 2024-11-07T12:42:42,422 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffbbe32019fd517ea21eab76e142aa7f/C in TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:42,422 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/7648d58a095140fa8ac33172dabcee71, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/96ebafeecff1423888688a233f19b19a, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/8da116fb23314cdebd8461f50d83a03a, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/1d106a1eacd74c869ae48dab987b4f81] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp, totalSize=45.9 K 2024-11-07T12:42:42,425 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7648d58a095140fa8ac33172dabcee71, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1730983359512 2024-11-07T12:42:42,425 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96ebafeecff1423888688a233f19b19a, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1730983360074 2024-11-07T12:42:42,425 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8da116fb23314cdebd8461f50d83a03a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1730983360321 2024-11-07T12:42:42,426 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d106a1eacd74c869ae48dab987b4f81, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1730983361035 2024-11-07T12:42:42,432 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-07T12:42:42,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:42,433 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing ffbbe32019fd517ea21eab76e142aa7f 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-11-07T12:42:42,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffbbe32019fd517ea21eab76e142aa7f, store=A 2024-11-07T12:42:42,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:42:42,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffbbe32019fd517ea21eab76e142aa7f, store=B 2024-11-07T12:42:42,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:42:42,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffbbe32019fd517ea21eab76e142aa7f, store=C 2024-11-07T12:42:42,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:42:42,434 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffbbe32019fd517ea21eab76e142aa7f#C#compaction#386 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:42:42,435 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/78e76d73414f4a7a981bdcf21bb49a07 is 50, key is test_row_0/C:col10/1730983361718/Put/seqid=0 2024-11-07T12:42:42,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742281_1457 (size=12881) 2024-11-07T12:42:42,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-07T12:42:42,450 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/78e76d73414f4a7a981bdcf21bb49a07 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/78e76d73414f4a7a981bdcf21bb49a07 2024-11-07T12:42:42,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411074d83c54d0f82455c955f02fefb659c1a_ffbbe32019fd517ea21eab76e142aa7f is 50, key is test_row_0/A:col10/1730983361820/Put/seqid=0 2024-11-07T12:42:42,461 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ffbbe32019fd517ea21eab76e142aa7f/C of ffbbe32019fd517ea21eab76e142aa7f into 78e76d73414f4a7a981bdcf21bb49a07(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:42:42,461 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffbbe32019fd517ea21eab76e142aa7f: 2024-11-07T12:42:42,461 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., storeName=ffbbe32019fd517ea21eab76e142aa7f/C, priority=12, startTime=1730983362297; duration=0sec 2024-11-07T12:42:42,461 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:42:42,461 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffbbe32019fd517ea21eab76e142aa7f:C 2024-11-07T12:42:42,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742282_1458 (size=12454) 2024-11-07T12:42:42,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:42,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. as already flushing 2024-11-07T12:42:42,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:42,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55862 deadline: 1730983422542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:42,553 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:42,553 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:42,553 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:42,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:42,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55856 deadline: 1730983422544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:42,560 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:42,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:42,560 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:42,560 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:42,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55858 deadline: 1730983422545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:42,560 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:42,560 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:42,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:42,560 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:42,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55828 deadline: 1730983422546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:42,560 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:42,560 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:42,560 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:42,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:42,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55844 deadline: 1730983422582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:42,595 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to db9ad1cb6cf9,34293,1730983260217 for 'test_row_2' in ffbbe32019fd517ea21eab76e142aa7f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2256 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:42:42,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:42,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:42,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:42,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:42,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55862 deadline: 1730983422661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:42,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:42,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55858 deadline: 1730983422662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:42,671 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:42,671 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:42,671 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:42,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:42,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:42,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:42,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:42,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55856 deadline: 1730983422672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:42,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:42,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55828 deadline: 1730983422673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:42,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:42,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:42,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:42,680 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:42,680 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:42,680 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:42,791 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/c3691d6c0606412d8710ee14052e779e as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/c3691d6c0606412d8710ee14052e779e 2024-11-07T12:42:42,804 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ffbbe32019fd517ea21eab76e142aa7f/B of ffbbe32019fd517ea21eab76e142aa7f into c3691d6c0606412d8710ee14052e779e(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:42:42,804 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffbbe32019fd517ea21eab76e142aa7f: 2024-11-07T12:42:42,804 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., storeName=ffbbe32019fd517ea21eab76e142aa7f/B, priority=12, startTime=1730983362297; duration=0sec 2024-11-07T12:42:42,804 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:42:42,804 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffbbe32019fd517ea21eab76e142aa7f:B 2024-11-07T12:42:42,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:42,871 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411074d83c54d0f82455c955f02fefb659c1a_ffbbe32019fd517ea21eab76e142aa7f to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411074d83c54d0f82455c955f02fefb659c1a_ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:42,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/dd7185696b644f2eae55aa1beff625ff, store: [table=TestAcidGuarantees family=A region=ffbbe32019fd517ea21eab76e142aa7f] 2024-11-07T12:42:42,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/dd7185696b644f2eae55aa1beff625ff is 175, key is test_row_0/A:col10/1730983361820/Put/seqid=0 2024-11-07T12:42:42,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742283_1459 (size=31255) 2024-11-07T12:42:42,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:42,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55858 deadline: 1730983422882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:42,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:42,891 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:42,891 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:42,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55862 deadline: 1730983422882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:42,892 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:42,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:42,892 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:42,892 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:42,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55856 deadline: 1730983422882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:42,892 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:42,892 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:42,892 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:42,892 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:42,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:42,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55828 deadline: 1730983422891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:42,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:42,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:42,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:42,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-07T12:42:43,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:43,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55856 deadline: 1730983423203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:43,225 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:43,225 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:43,225 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:43,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:43,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55828 deadline: 1730983423203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:43,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:43,226 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:43,226 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:43,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55858 deadline: 1730983423205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:43,226 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:43,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:43,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55862 deadline: 1730983423207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:43,227 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:43,227 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:43,227 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:43,228 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:43,228 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:43,229 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:43,277 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=331, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/dd7185696b644f2eae55aa1beff625ff 2024-11-07T12:42:43,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/994877357ab24ace88cf7263cccf090c is 50, key is test_row_0/B:col10/1730983361820/Put/seqid=0 2024-11-07T12:42:43,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742284_1460 (size=12301) 2024-11-07T12:42:43,727 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/994877357ab24ace88cf7263cccf090c 2024-11-07T12:42:43,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:43,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55856 deadline: 1730983423734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:43,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:43,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55828 deadline: 1730983423735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:43,741 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:43,741 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:43,741 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:43,741 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:43,741 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:43,741 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:43,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/efef0128b57f4b1690189720f4e5eb94 is 50, key is test_row_0/C:col10/1730983361820/Put/seqid=0 2024-11-07T12:42:43,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:43,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55862 deadline: 1730983423748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:43,758 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:43,758 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:43,758 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:43,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:43,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55858 deadline: 1730983423748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:43,759 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:43,759 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:43,759 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:43,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742285_1461 (size=12301) 2024-11-07T12:42:43,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-07T12:42:44,174 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/efef0128b57f4b1690189720f4e5eb94 2024-11-07T12:42:44,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/dd7185696b644f2eae55aa1beff625ff as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/dd7185696b644f2eae55aa1beff625ff 2024-11-07T12:42:44,195 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/dd7185696b644f2eae55aa1beff625ff, entries=150, sequenceid=331, filesize=30.5 K 2024-11-07T12:42:44,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/994877357ab24ace88cf7263cccf090c as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/994877357ab24ace88cf7263cccf090c 2024-11-07T12:42:44,205 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/994877357ab24ace88cf7263cccf090c, entries=150, sequenceid=331, filesize=12.0 K 2024-11-07T12:42:44,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-07T12:42:44,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/efef0128b57f4b1690189720f4e5eb94 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/efef0128b57f4b1690189720f4e5eb94 2024-11-07T12:42:44,220 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/efef0128b57f4b1690189720f4e5eb94, entries=150, sequenceid=331, filesize=12.0 K 2024-11-07T12:42:44,225 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=53.67 KB/54960 for ffbbe32019fd517ea21eab76e142aa7f in 1792ms, sequenceid=331, compaction requested=false 2024-11-07T12:42:44,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for ffbbe32019fd517ea21eab76e142aa7f: 2024-11-07T12:42:44,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:44,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-07T12:42:44,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-07T12:42:44,230 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-07T12:42:44,230 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4120 sec 2024-11-07T12:42:44,232 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 2.4190 sec 2024-11-07T12:42:44,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:44,615 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ffbbe32019fd517ea21eab76e142aa7f 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-07T12:42:44,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffbbe32019fd517ea21eab76e142aa7f, store=A 2024-11-07T12:42:44,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:44,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffbbe32019fd517ea21eab76e142aa7f, store=B 2024-11-07T12:42:44,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:44,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffbbe32019fd517ea21eab76e142aa7f, store=C 2024-11-07T12:42:44,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:44,628 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107e89bba87e88a4faf9129c0f87ac1a5e7_ffbbe32019fd517ea21eab76e142aa7f is 50, key is test_row_0/A:col10/1730983362544/Put/seqid=0 2024-11-07T12:42:44,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742286_1462 (size=14994) 2024-11-07T12:42:44,646 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:44,651 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107e89bba87e88a4faf9129c0f87ac1a5e7_ffbbe32019fd517ea21eab76e142aa7f to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107e89bba87e88a4faf9129c0f87ac1a5e7_ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:44,653 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/360be99d55d84b7eafd6fdb6cd034dbf, store: [table=TestAcidGuarantees family=A region=ffbbe32019fd517ea21eab76e142aa7f] 2024-11-07T12:42:44,653 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/360be99d55d84b7eafd6fdb6cd034dbf is 175, key is test_row_0/A:col10/1730983362544/Put/seqid=0 2024-11-07T12:42:44,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742287_1463 (size=39949) 2024-11-07T12:42:44,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:44,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55844 deadline: 1730983424755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:44,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:44,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:44,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55856 deadline: 1730983424764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:44,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:44,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:44,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:44,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:44,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:44,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:44,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55828 deadline: 1730983424766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:44,778 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:44,778 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:44,778 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:44,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:44,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55862 deadline: 1730983424772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:44,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:44,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55858 deadline: 1730983424772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:44,781 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to db9ad1cb6cf9,34293,1730983260217 for 'test_row_1' in ffbbe32019fd517ea21eab76e142aa7f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2239 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:42:44,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:44,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:44,782 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to db9ad1cb6cf9,34293,1730983260217 for 'test_row_2' in ffbbe32019fd517ea21eab76e142aa7f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2236 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:42:44,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:44,782 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:44,782 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:44,782 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:44,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:44,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55856 deadline: 1730983424882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:44,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:44,892 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:44,892 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:44,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55844 deadline: 1730983424882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:44,892 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:44,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:44,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:44,893 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:44,893 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:44,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55828 deadline: 1730983424882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:44,894 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:44,894 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:44,894 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:45,065 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=345, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/360be99d55d84b7eafd6fdb6cd034dbf 2024-11-07T12:42:45,073 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/9b9b38f97ebd47fe9d7e258f5800eaf0 is 50, key is test_row_0/B:col10/1730983362544/Put/seqid=0 2024-11-07T12:42:45,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742288_1464 (size=12301) 2024-11-07T12:42:45,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:45,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55844 deadline: 1730983425101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:45,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:45,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:45,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55856 deadline: 1730983425102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:45,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:45,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:45,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:45,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55828 deadline: 1730983425102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:45,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:45,113 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:45,113 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:45,113 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:45,113 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:45,113 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:45,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:45,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55844 deadline: 1730983425422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:45,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:45,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55856 deadline: 1730983425422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:45,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:45,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55828 deadline: 1730983425422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:45,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:45,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:45,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:45,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:45,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:45,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:45,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:45,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:45,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:45,485 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/9b9b38f97ebd47fe9d7e258f5800eaf0 2024-11-07T12:42:45,505 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/577fe71a1f34454084f0fec48cbc1fec is 50, key is test_row_0/C:col10/1730983362544/Put/seqid=0 2024-11-07T12:42:45,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742289_1465 (size=12301) 2024-11-07T12:42:45,529 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/577fe71a1f34454084f0fec48cbc1fec 2024-11-07T12:42:45,535 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/360be99d55d84b7eafd6fdb6cd034dbf as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/360be99d55d84b7eafd6fdb6cd034dbf 2024-11-07T12:42:45,538 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/360be99d55d84b7eafd6fdb6cd034dbf, entries=200, sequenceid=345, filesize=39.0 K 2024-11-07T12:42:45,540 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/9b9b38f97ebd47fe9d7e258f5800eaf0 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/9b9b38f97ebd47fe9d7e258f5800eaf0 2024-11-07T12:42:45,549 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/9b9b38f97ebd47fe9d7e258f5800eaf0, entries=150, sequenceid=345, filesize=12.0 K 2024-11-07T12:42:45,550 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/577fe71a1f34454084f0fec48cbc1fec as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/577fe71a1f34454084f0fec48cbc1fec 2024-11-07T12:42:45,571 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/577fe71a1f34454084f0fec48cbc1fec, entries=150, sequenceid=345, filesize=12.0 K 2024-11-07T12:42:45,572 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for ffbbe32019fd517ea21eab76e142aa7f in 958ms, sequenceid=345, compaction requested=true 2024-11-07T12:42:45,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ffbbe32019fd517ea21eab76e142aa7f: 2024-11-07T12:42:45,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffbbe32019fd517ea21eab76e142aa7f:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:42:45,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:42:45,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffbbe32019fd517ea21eab76e142aa7f:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:42:45,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-07T12:42:45,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffbbe32019fd517ea21eab76e142aa7f:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:42:45,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-07T12:42:45,572 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:42:45,572 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:42:45,581 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:42:45,581 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): ffbbe32019fd517ea21eab76e142aa7f/C is initiating minor compaction (all files) 2024-11-07T12:42:45,581 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffbbe32019fd517ea21eab76e142aa7f/C in TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:45,581 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/78e76d73414f4a7a981bdcf21bb49a07, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/efef0128b57f4b1690189720f4e5eb94, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/577fe71a1f34454084f0fec48cbc1fec] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp, totalSize=36.6 K 2024-11-07T12:42:45,582 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103039 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:42:45,582 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): ffbbe32019fd517ea21eab76e142aa7f/A is initiating minor compaction (all files) 2024-11-07T12:42:45,582 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffbbe32019fd517ea21eab76e142aa7f/A in TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:45,582 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/553fb7f407c74aff9cbc4445fc96f9e2, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/dd7185696b644f2eae55aa1beff625ff, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/360be99d55d84b7eafd6fdb6cd034dbf] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp, totalSize=100.6 K 2024-11-07T12:42:45,582 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:45,582 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. files: [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/553fb7f407c74aff9cbc4445fc96f9e2, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/dd7185696b644f2eae55aa1beff625ff, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/360be99d55d84b7eafd6fdb6cd034dbf] 2024-11-07T12:42:45,582 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 78e76d73414f4a7a981bdcf21bb49a07, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1730983361035 2024-11-07T12:42:45,586 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 553fb7f407c74aff9cbc4445fc96f9e2, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1730983361035 2024-11-07T12:42:45,587 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting efef0128b57f4b1690189720f4e5eb94, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1730983361820 2024-11-07T12:42:45,588 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd7185696b644f2eae55aa1beff625ff, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1730983361820 2024-11-07T12:42:45,588 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 577fe71a1f34454084f0fec48cbc1fec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1730983362541 2024-11-07T12:42:45,588 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 360be99d55d84b7eafd6fdb6cd034dbf, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1730983362541 2024-11-07T12:42:45,603 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffbbe32019fd517ea21eab76e142aa7f#C#compaction#393 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:42:45,604 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/253b14d46edc47bcaf6ef0e89edb8f0d is 50, key is test_row_0/C:col10/1730983362544/Put/seqid=0 2024-11-07T12:42:45,613 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ffbbe32019fd517ea21eab76e142aa7f] 2024-11-07T12:42:45,625 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411077c310691064d49dc998197b4d6d69d8d_ffbbe32019fd517ea21eab76e142aa7f store=[table=TestAcidGuarantees family=A region=ffbbe32019fd517ea21eab76e142aa7f] 2024-11-07T12:42:45,627 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411077c310691064d49dc998197b4d6d69d8d_ffbbe32019fd517ea21eab76e142aa7f, store=[table=TestAcidGuarantees family=A region=ffbbe32019fd517ea21eab76e142aa7f] 2024-11-07T12:42:45,628 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411077c310691064d49dc998197b4d6d69d8d_ffbbe32019fd517ea21eab76e142aa7f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ffbbe32019fd517ea21eab76e142aa7f] 2024-11-07T12:42:45,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742290_1466 (size=12983) 2024-11-07T12:42:45,664 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/253b14d46edc47bcaf6ef0e89edb8f0d as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/253b14d46edc47bcaf6ef0e89edb8f0d 2024-11-07T12:42:45,674 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ffbbe32019fd517ea21eab76e142aa7f/C of ffbbe32019fd517ea21eab76e142aa7f into 253b14d46edc47bcaf6ef0e89edb8f0d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:42:45,674 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffbbe32019fd517ea21eab76e142aa7f: 2024-11-07T12:42:45,674 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., storeName=ffbbe32019fd517ea21eab76e142aa7f/C, priority=13, startTime=1730983365572; duration=0sec 2024-11-07T12:42:45,674 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:42:45,674 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffbbe32019fd517ea21eab76e142aa7f:C 2024-11-07T12:42:45,674 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:42:45,675 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:42:45,675 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): ffbbe32019fd517ea21eab76e142aa7f/B is initiating minor compaction (all files) 2024-11-07T12:42:45,675 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffbbe32019fd517ea21eab76e142aa7f/B in TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:45,675 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/c3691d6c0606412d8710ee14052e779e, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/994877357ab24ace88cf7263cccf090c, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/9b9b38f97ebd47fe9d7e258f5800eaf0] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp, totalSize=36.6 K 2024-11-07T12:42:45,676 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting c3691d6c0606412d8710ee14052e779e, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1730983361035 2024-11-07T12:42:45,676 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 994877357ab24ace88cf7263cccf090c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1730983361820 2024-11-07T12:42:45,676 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b9b38f97ebd47fe9d7e258f5800eaf0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1730983362541 2024-11-07T12:42:45,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742291_1467 (size=4469) 2024-11-07T12:42:45,705 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffbbe32019fd517ea21eab76e142aa7f#B#compaction#395 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:42:45,705 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/c3a89f91f97e431b86409c2e08f824a3 is 50, key is test_row_0/B:col10/1730983362544/Put/seqid=0 2024-11-07T12:42:45,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742292_1468 (size=12983) 2024-11-07T12:42:45,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:45,946 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ffbbe32019fd517ea21eab76e142aa7f 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-07T12:42:45,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffbbe32019fd517ea21eab76e142aa7f, store=A 2024-11-07T12:42:45,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:42:45,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffbbe32019fd517ea21eab76e142aa7f, store=B 2024-11-07T12:42:45,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:42:45,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffbbe32019fd517ea21eab76e142aa7f, store=C 2024-11-07T12:42:45,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:42:45,958 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107bb7cc13f77ed4875abf23d5ea3df69b8_ffbbe32019fd517ea21eab76e142aa7f is 50, key is test_row_0/A:col10/1730983364715/Put/seqid=0 2024-11-07T12:42:45,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742293_1469 (size=14994) 2024-11-07T12:42:45,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-07T12:42:45,972 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:42:45,972 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:42:45,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-07T12:42:45,976 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:42:45,978 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:42:45,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-07T12:42:45,978 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:42:45,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:45,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55844 deadline: 1730983425971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:45,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:45,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:45,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:45,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:45,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55828 deadline: 1730983425973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:45,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:45,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:45,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:45,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:45,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55856 deadline: 1730983425991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:45,996 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:45,996 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:45,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:46,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-07T12:42:46,094 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffbbe32019fd517ea21eab76e142aa7f#A#compaction#394 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:42:46,094 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/c6d49ce7c5604aecb3f86cd10ed6e871 is 175, key is test_row_0/A:col10/1730983362544/Put/seqid=0 2024-11-07T12:42:46,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742294_1470 (size=31937) 2024-11-07T12:42:46,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:46,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55828 deadline: 1730983426101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:46,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:46,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:46,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55856 deadline: 1730983426102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:46,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:46,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:46,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:46,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:46,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:46,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55844 deadline: 1730983426102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:46,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:46,115 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:46,115 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:46,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:46,133 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-07T12:42:46,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:46,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. as already flushing 2024-11-07T12:42:46,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:46,134 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:46,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:46,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:46,146 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/c6d49ce7c5604aecb3f86cd10ed6e871 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/c6d49ce7c5604aecb3f86cd10ed6e871 2024-11-07T12:42:46,146 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/c3a89f91f97e431b86409c2e08f824a3 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/c3a89f91f97e431b86409c2e08f824a3 2024-11-07T12:42:46,152 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ffbbe32019fd517ea21eab76e142aa7f/B of ffbbe32019fd517ea21eab76e142aa7f into c3a89f91f97e431b86409c2e08f824a3(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:42:46,152 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffbbe32019fd517ea21eab76e142aa7f: 2024-11-07T12:42:46,152 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., storeName=ffbbe32019fd517ea21eab76e142aa7f/B, priority=13, startTime=1730983365572; duration=0sec 2024-11-07T12:42:46,152 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:42:46,152 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffbbe32019fd517ea21eab76e142aa7f:B 2024-11-07T12:42:46,156 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ffbbe32019fd517ea21eab76e142aa7f/A of ffbbe32019fd517ea21eab76e142aa7f into c6d49ce7c5604aecb3f86cd10ed6e871(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:42:46,156 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffbbe32019fd517ea21eab76e142aa7f: 2024-11-07T12:42:46,156 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., storeName=ffbbe32019fd517ea21eab76e142aa7f/A, priority=13, startTime=1730983365572; duration=0sec 2024-11-07T12:42:46,156 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:42:46,156 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffbbe32019fd517ea21eab76e142aa7f:A 2024-11-07T12:42:46,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-07T12:42:46,294 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-07T12:42:46,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:46,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. as already flushing 2024-11-07T12:42:46,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:46,294 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:46,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:46,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:46,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:46,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55828 deadline: 1730983426321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:46,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:46,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:46,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:46,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:46,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55844 deadline: 1730983426322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:46,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:46,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:46,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:46,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:46,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55856 deadline: 1730983426322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:46,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:46,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:46,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:46,364 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:46,368 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107bb7cc13f77ed4875abf23d5ea3df69b8_ffbbe32019fd517ea21eab76e142aa7f to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107bb7cc13f77ed4875abf23d5ea3df69b8_ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:46,368 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/e27545bdfa8e4303b2b559f30a96ee87, store: [table=TestAcidGuarantees family=A region=ffbbe32019fd517ea21eab76e142aa7f] 2024-11-07T12:42:46,369 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/e27545bdfa8e4303b2b559f30a96ee87 is 175, key is test_row_0/A:col10/1730983364715/Put/seqid=0 2024-11-07T12:42:46,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742295_1471 (size=39949) 2024-11-07T12:42:46,386 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=367, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/e27545bdfa8e4303b2b559f30a96ee87 2024-11-07T12:42:46,403 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/72e7104b418b4372adcf83e97eced795 is 50, key is test_row_0/B:col10/1730983364715/Put/seqid=0 2024-11-07T12:42:46,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742296_1472 (size=12301) 2024-11-07T12:42:46,432 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/72e7104b418b4372adcf83e97eced795 2024-11-07T12:42:46,448 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-07T12:42:46,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:46,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. as already flushing 2024-11-07T12:42:46,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:46,450 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:46,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:46,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:46,455 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/88efdea78d8442b5a7aea59cc8b10a98 is 50, key is test_row_0/C:col10/1730983364715/Put/seqid=0 2024-11-07T12:42:46,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742297_1473 (size=12301) 2024-11-07T12:42:46,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-07T12:42:46,603 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-07T12:42:46,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:46,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. as already flushing 2024-11-07T12:42:46,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:46,603 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:46,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:46,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:46,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:46,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55828 deadline: 1730983426642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:46,649 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:46,649 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:46,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:46,649 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:46,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55844 deadline: 1730983426642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:46,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:46,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:46,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55856 deadline: 1730983426642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:46,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:46,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:46,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:46,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:46,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:46,756 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-07T12:42:46,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:46,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. as already flushing 2024-11-07T12:42:46,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:46,757 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:46,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:46,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:46,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:46,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55862 deadline: 1730983426792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:46,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:46,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55858 deadline: 1730983426792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:46,799 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to db9ad1cb6cf9,34293,1730983260217 for 'test_row_2' in ffbbe32019fd517ea21eab76e142aa7f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4254 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:42:46,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:46,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:46,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:46,800 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to db9ad1cb6cf9,34293,1730983260217 for 'test_row_1' in ffbbe32019fd517ea21eab76e142aa7f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4258 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:42:46,801 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:46,801 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:46,801 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:46,876 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/88efdea78d8442b5a7aea59cc8b10a98 2024-11-07T12:42:46,880 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/e27545bdfa8e4303b2b559f30a96ee87 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/e27545bdfa8e4303b2b559f30a96ee87 2024-11-07T12:42:46,887 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/e27545bdfa8e4303b2b559f30a96ee87, entries=200, sequenceid=367, filesize=39.0 K 2024-11-07T12:42:46,887 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/72e7104b418b4372adcf83e97eced795 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/72e7104b418b4372adcf83e97eced795 2024-11-07T12:42:46,891 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/72e7104b418b4372adcf83e97eced795, entries=150, sequenceid=367, filesize=12.0 K 2024-11-07T12:42:46,891 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/88efdea78d8442b5a7aea59cc8b10a98 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/88efdea78d8442b5a7aea59cc8b10a98 2024-11-07T12:42:46,895 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/88efdea78d8442b5a7aea59cc8b10a98, entries=150, sequenceid=367, filesize=12.0 K 2024-11-07T12:42:46,896 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for ffbbe32019fd517ea21eab76e142aa7f in 950ms, sequenceid=367, compaction requested=false 2024-11-07T12:42:46,896 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ffbbe32019fd517ea21eab76e142aa7f: 2024-11-07T12:42:46,912 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-07T12:42:46,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:46,913 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing ffbbe32019fd517ea21eab76e142aa7f 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-07T12:42:46,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffbbe32019fd517ea21eab76e142aa7f, store=A 2024-11-07T12:42:46,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:46,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffbbe32019fd517ea21eab76e142aa7f, store=B 2024-11-07T12:42:46,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:46,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffbbe32019fd517ea21eab76e142aa7f, store=C 2024-11-07T12:42:46,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:46,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411070f2868b06a7f4cacab30bbfb1d0df4c8_ffbbe32019fd517ea21eab76e142aa7f is 50, key is test_row_0/A:col10/1730983365967/Put/seqid=0 2024-11-07T12:42:46,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742298_1474 (size=12454) 2024-11-07T12:42:47,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-07T12:42:47,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:47,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. as already flushing 2024-11-07T12:42:47,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:47,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55828 deadline: 1730983427220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:47,235 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:47,235 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:47,236 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:47,245 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:47,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55844 deadline: 1730983427234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:47,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:47,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:47,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:47,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:47,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55856 deadline: 1730983427235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:47,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:47,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:47,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:47,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:47,342 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411070f2868b06a7f4cacab30bbfb1d0df4c8_ffbbe32019fd517ea21eab76e142aa7f to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411070f2868b06a7f4cacab30bbfb1d0df4c8_ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:47,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/240ca42cbde849edae557294bd9f06fc, store: [table=TestAcidGuarantees family=A region=ffbbe32019fd517ea21eab76e142aa7f] 2024-11-07T12:42:47,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/240ca42cbde849edae557294bd9f06fc is 175, key is test_row_0/A:col10/1730983365967/Put/seqid=0 2024-11-07T12:42:47,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:47,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55828 deadline: 1730983427341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:47,355 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:47,355 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:47,355 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:47,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742299_1475 (size=31255) 2024-11-07T12:42:47,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:47,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55844 deadline: 1730983427352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:47,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:47,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:47,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:47,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:47,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55856 deadline: 1730983427362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:47,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:47,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:47,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:47,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:47,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55828 deadline: 1730983427561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:47,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:47,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:47,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:47,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:47,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55844 deadline: 1730983427571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:47,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:47,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:47,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:47,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:47,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55856 deadline: 1730983427582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:47,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:47,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:47,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:47,762 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=380, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/240ca42cbde849edae557294bd9f06fc 2024-11-07T12:42:47,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/a590c3c3e280452ba75352d5adcc0355 is 50, key is test_row_0/B:col10/1730983365967/Put/seqid=0 2024-11-07T12:42:47,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742300_1476 (size=12301) 2024-11-07T12:42:47,785 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/a590c3c3e280452ba75352d5adcc0355 2024-11-07T12:42:47,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/66365c60cf6f4df2b5c318e89a9eb16d is 50, key is test_row_0/C:col10/1730983365967/Put/seqid=0 2024-11-07T12:42:47,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742301_1477 (size=12301) 2024-11-07T12:42:47,810 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/66365c60cf6f4df2b5c318e89a9eb16d 2024-11-07T12:42:47,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/240ca42cbde849edae557294bd9f06fc as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/240ca42cbde849edae557294bd9f06fc 2024-11-07T12:42:47,819 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/240ca42cbde849edae557294bd9f06fc, entries=150, sequenceid=380, filesize=30.5 K 2024-11-07T12:42:47,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/a590c3c3e280452ba75352d5adcc0355 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/a590c3c3e280452ba75352d5adcc0355 2024-11-07T12:42:47,825 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/a590c3c3e280452ba75352d5adcc0355, entries=150, sequenceid=380, filesize=12.0 K 2024-11-07T12:42:47,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/66365c60cf6f4df2b5c318e89a9eb16d as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/66365c60cf6f4df2b5c318e89a9eb16d 2024-11-07T12:42:47,841 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/66365c60cf6f4df2b5c318e89a9eb16d, entries=150, sequenceid=380, filesize=12.0 K 2024-11-07T12:42:47,845 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=120.76 KB/123660 for ffbbe32019fd517ea21eab76e142aa7f in 931ms, sequenceid=380, compaction requested=true 2024-11-07T12:42:47,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for ffbbe32019fd517ea21eab76e142aa7f: 2024-11-07T12:42:47,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:47,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-07T12:42:47,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-07T12:42:47,858 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-07T12:42:47,858 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8780 sec 2024-11-07T12:42:47,865 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 1.8870 sec 2024-11-07T12:42:47,878 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ffbbe32019fd517ea21eab76e142aa7f 3/3 column families, dataSize=127.47 KB heapSize=379.97 KB 2024-11-07T12:42:47,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffbbe32019fd517ea21eab76e142aa7f, store=A 2024-11-07T12:42:47,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:42:47,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffbbe32019fd517ea21eab76e142aa7f, store=B 2024-11-07T12:42:47,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:42:47,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffbbe32019fd517ea21eab76e142aa7f, store=C 2024-11-07T12:42:47,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:42:47,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:47,910 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110751b9948ca1ef47438149e92e17a0c2c9_ffbbe32019fd517ea21eab76e142aa7f is 50, key is test_row_0/A:col10/1730983367877/Put/seqid=0 2024-11-07T12:42:47,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742302_1478 (size=14994) 2024-11-07T12:42:47,919 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:47,926 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110751b9948ca1ef47438149e92e17a0c2c9_ffbbe32019fd517ea21eab76e142aa7f to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110751b9948ca1ef47438149e92e17a0c2c9_ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:47,927 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/8fc93f8cb7894943aff8e090cecd35a3, store: [table=TestAcidGuarantees family=A region=ffbbe32019fd517ea21eab76e142aa7f] 2024-11-07T12:42:47,927 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/8fc93f8cb7894943aff8e090cecd35a3 is 175, key is test_row_0/A:col10/1730983367877/Put/seqid=0 2024-11-07T12:42:47,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:47,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55856 deadline: 1730983427911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:47,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:47,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:47,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55828 deadline: 1730983427912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:47,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:47,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:47,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:47,930 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:47,930 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:47,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55844 deadline: 1730983427913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:47,930 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:47,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:47,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:47,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:47,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742303_1479 (size=39949) 2024-11-07T12:42:48,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:48,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55856 deadline: 1730983428042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:48,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:48,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55828 deadline: 1730983428045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:48,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:48,055 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:48,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55844 deadline: 1730983428045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:48,055 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:48,055 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:48,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:48,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:48,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:48,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:48,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:48,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:48,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-07T12:42:48,122 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:42:48,123 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:42:48,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-07T12:42:48,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-07T12:42:48,125 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:42:48,126 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:42:48,126 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:42:48,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-07T12:42:48,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:48,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55844 deadline: 1730983428261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:48,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:48,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:48,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:48,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:48,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55828 deadline: 1730983428261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:48,275 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:48,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:48,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:48,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:48,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55856 deadline: 1730983428262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:48,276 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:48,276 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:48,276 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:48,279 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-07T12:42:48,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:48,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. as already flushing 2024-11-07T12:42:48,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:48,279 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:48,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:48,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:48,350 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=402, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/8fc93f8cb7894943aff8e090cecd35a3 2024-11-07T12:42:48,356 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/1a34d5b7ceaa4bf59bcf202ae7f1cab3 is 50, key is test_row_0/B:col10/1730983367877/Put/seqid=0 2024-11-07T12:42:48,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742304_1480 (size=12301) 2024-11-07T12:42:48,367 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/1a34d5b7ceaa4bf59bcf202ae7f1cab3 2024-11-07T12:42:48,378 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/9f0feb6f84d4477997d169b50d196cdd is 50, key is test_row_0/C:col10/1730983367877/Put/seqid=0 2024-11-07T12:42:48,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742305_1481 (size=12301) 2024-11-07T12:42:48,432 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-07T12:42:48,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:48,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. as already flushing 2024-11-07T12:42:48,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:48,433 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:48,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:48,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:48,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-07T12:42:48,586 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-07T12:42:48,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:48,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. as already flushing 2024-11-07T12:42:48,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:48,587 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:48,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:48,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:48,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:48,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55844 deadline: 1730983428582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:48,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:48,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:48,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:48,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55828 deadline: 1730983428582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:48,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:48,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:48,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:48,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:48,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55856 deadline: 1730983428582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:48,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:48,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:48,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:48,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:48,740 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-07T12:42:48,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:48,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. as already flushing 2024-11-07T12:42:48,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:48,741 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:48,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:48,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:48,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-07T12:42:48,798 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/9f0feb6f84d4477997d169b50d196cdd 2024-11-07T12:42:48,802 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/8fc93f8cb7894943aff8e090cecd35a3 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/8fc93f8cb7894943aff8e090cecd35a3 2024-11-07T12:42:48,806 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/8fc93f8cb7894943aff8e090cecd35a3, entries=200, sequenceid=402, filesize=39.0 K 2024-11-07T12:42:48,807 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/1a34d5b7ceaa4bf59bcf202ae7f1cab3 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/1a34d5b7ceaa4bf59bcf202ae7f1cab3 2024-11-07T12:42:48,811 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/1a34d5b7ceaa4bf59bcf202ae7f1cab3, entries=150, sequenceid=402, filesize=12.0 K 2024-11-07T12:42:48,812 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/9f0feb6f84d4477997d169b50d196cdd as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/9f0feb6f84d4477997d169b50d196cdd 2024-11-07T12:42:48,816 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/9f0feb6f84d4477997d169b50d196cdd, entries=150, sequenceid=402, filesize=12.0 K 2024-11-07T12:42:48,817 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~379.92 KB/389040, currentSize=46.96 KB/48090 for ffbbe32019fd517ea21eab76e142aa7f in 939ms, sequenceid=402, compaction requested=true 2024-11-07T12:42:48,817 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ffbbe32019fd517ea21eab76e142aa7f: 2024-11-07T12:42:48,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffbbe32019fd517ea21eab76e142aa7f:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:42:48,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:42:48,817 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:42:48,817 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:42:48,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffbbe32019fd517ea21eab76e142aa7f:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:42:48,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:42:48,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffbbe32019fd517ea21eab76e142aa7f:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:42:48,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:42:48,818 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 143090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:42:48,818 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:42:48,818 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): ffbbe32019fd517ea21eab76e142aa7f/A is initiating minor compaction (all files) 2024-11-07T12:42:48,818 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): ffbbe32019fd517ea21eab76e142aa7f/B is initiating minor compaction (all files) 2024-11-07T12:42:48,818 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffbbe32019fd517ea21eab76e142aa7f/A in TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:48,818 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffbbe32019fd517ea21eab76e142aa7f/B in TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:48,819 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/c6d49ce7c5604aecb3f86cd10ed6e871, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/e27545bdfa8e4303b2b559f30a96ee87, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/240ca42cbde849edae557294bd9f06fc, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/8fc93f8cb7894943aff8e090cecd35a3] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp, totalSize=139.7 K 2024-11-07T12:42:48,819 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/c3a89f91f97e431b86409c2e08f824a3, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/72e7104b418b4372adcf83e97eced795, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/a590c3c3e280452ba75352d5adcc0355, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/1a34d5b7ceaa4bf59bcf202ae7f1cab3] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp, totalSize=48.7 K 2024-11-07T12:42:48,819 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:48,819 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. files: [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/c6d49ce7c5604aecb3f86cd10ed6e871, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/e27545bdfa8e4303b2b559f30a96ee87, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/240ca42cbde849edae557294bd9f06fc, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/8fc93f8cb7894943aff8e090cecd35a3] 2024-11-07T12:42:48,819 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting c3a89f91f97e431b86409c2e08f824a3, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1730983362541 2024-11-07T12:42:48,819 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6d49ce7c5604aecb3f86cd10ed6e871, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1730983362541 2024-11-07T12:42:48,819 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 72e7104b418b4372adcf83e97eced795, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1730983364715 2024-11-07T12:42:48,819 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting e27545bdfa8e4303b2b559f30a96ee87, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1730983364715 2024-11-07T12:42:48,820 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 240ca42cbde849edae557294bd9f06fc, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1730983365967 2024-11-07T12:42:48,820 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting a590c3c3e280452ba75352d5adcc0355, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1730983365967 2024-11-07T12:42:48,820 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a34d5b7ceaa4bf59bcf202ae7f1cab3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1730983367200 2024-11-07T12:42:48,820 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8fc93f8cb7894943aff8e090cecd35a3, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1730983367200 2024-11-07T12:42:48,830 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ffbbe32019fd517ea21eab76e142aa7f] 2024-11-07T12:42:48,831 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffbbe32019fd517ea21eab76e142aa7f#B#compaction#405 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:42:48,832 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/b3632077c45442cfa4a6b6a5cc1ab324 is 50, key is test_row_0/B:col10/1730983367877/Put/seqid=0 2024-11-07T12:42:48,835 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241107030e099cb2824a7a85a9add65aeb2a6c_ffbbe32019fd517ea21eab76e142aa7f store=[table=TestAcidGuarantees family=A region=ffbbe32019fd517ea21eab76e142aa7f] 2024-11-07T12:42:48,838 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241107030e099cb2824a7a85a9add65aeb2a6c_ffbbe32019fd517ea21eab76e142aa7f, store=[table=TestAcidGuarantees family=A region=ffbbe32019fd517ea21eab76e142aa7f] 2024-11-07T12:42:48,838 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107030e099cb2824a7a85a9add65aeb2a6c_ffbbe32019fd517ea21eab76e142aa7f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ffbbe32019fd517ea21eab76e142aa7f] 2024-11-07T12:42:48,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742306_1482 (size=13119) 2024-11-07T12:42:48,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742307_1483 (size=4469) 2024-11-07T12:42:48,894 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-07T12:42:48,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:48,894 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing ffbbe32019fd517ea21eab76e142aa7f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-07T12:42:48,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffbbe32019fd517ea21eab76e142aa7f, store=A 2024-11-07T12:42:48,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:48,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffbbe32019fd517ea21eab76e142aa7f, store=B 2024-11-07T12:42:48,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:48,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffbbe32019fd517ea21eab76e142aa7f, store=C 2024-11-07T12:42:48,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:48,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110776cb73b25c054f50be2d9827a410abfe_ffbbe32019fd517ea21eab76e142aa7f is 50, key is test_row_0/A:col10/1730983367912/Put/seqid=0 2024-11-07T12:42:48,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742308_1484 (size=12454) 2024-11-07T12:42:48,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:48,920 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110776cb73b25c054f50be2d9827a410abfe_ffbbe32019fd517ea21eab76e142aa7f to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110776cb73b25c054f50be2d9827a410abfe_ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:48,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/28a37e937a92412bb189a2b842aff785, store: [table=TestAcidGuarantees family=A region=ffbbe32019fd517ea21eab76e142aa7f] 2024-11-07T12:42:48,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/28a37e937a92412bb189a2b842aff785 is 175, key is test_row_0/A:col10/1730983367912/Put/seqid=0 2024-11-07T12:42:48,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742309_1485 (size=31255) 2024-11-07T12:42:48,932 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=412, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/28a37e937a92412bb189a2b842aff785 2024-11-07T12:42:48,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/825328eefa2243fd8974c98aefee7946 is 50, key is test_row_0/B:col10/1730983367912/Put/seqid=0 2024-11-07T12:42:48,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742310_1486 (size=12301) 2024-11-07T12:42:48,962 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/825328eefa2243fd8974c98aefee7946 2024-11-07T12:42:48,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/65888d63555a46a69abe47c1e8c64ea0 is 50, key is test_row_0/C:col10/1730983367912/Put/seqid=0 2024-11-07T12:42:48,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742311_1487 (size=12301) 2024-11-07T12:42:49,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:49,112 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. as already flushing 2024-11-07T12:42:49,193 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1730983369170, max timestamp=1730983369190Num uniques 859; 2024-11-07T12:42:49,193 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1730983369170, max timestamp=1730983369190Num uniques 859; 2024-11-07T12:42:49,193 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1730983369170, max timestamp=1730983369190Num uniques 859; 2024-11-07T12:42:49,194 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-07T12:42:49,194 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-07T12:42:49,194 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-07T12:42:49,251 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/b3632077c45442cfa4a6b6a5cc1ab324 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/b3632077c45442cfa4a6b6a5cc1ab324 2024-11-07T12:42:49,254 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffbbe32019fd517ea21eab76e142aa7f#A#compaction#406 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:42:49,255 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ffbbe32019fd517ea21eab76e142aa7f/B of ffbbe32019fd517ea21eab76e142aa7f into b3632077c45442cfa4a6b6a5cc1ab324(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:42:49,255 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffbbe32019fd517ea21eab76e142aa7f: 2024-11-07T12:42:49,255 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/cc68a3e0dd274d498844ee92198dbd92 is 175, key is test_row_0/A:col10/1730983367877/Put/seqid=0 2024-11-07T12:42:49,255 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., storeName=ffbbe32019fd517ea21eab76e142aa7f/B, priority=12, startTime=1730983368817; duration=0sec 2024-11-07T12:42:49,255 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:42:49,255 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffbbe32019fd517ea21eab76e142aa7f:B 2024-11-07T12:42:49,255 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:42:49,257 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:42:49,257 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): ffbbe32019fd517ea21eab76e142aa7f/C is initiating minor compaction (all files) 2024-11-07T12:42:49,257 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffbbe32019fd517ea21eab76e142aa7f/C in TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:49,257 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/253b14d46edc47bcaf6ef0e89edb8f0d, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/88efdea78d8442b5a7aea59cc8b10a98, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/66365c60cf6f4df2b5c318e89a9eb16d, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/9f0feb6f84d4477997d169b50d196cdd] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp, totalSize=48.7 K 2024-11-07T12:42:49,258 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 253b14d46edc47bcaf6ef0e89edb8f0d, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1730983362541 2024-11-07T12:42:49,258 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 88efdea78d8442b5a7aea59cc8b10a98, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1730983364715 2024-11-07T12:42:49,258 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 66365c60cf6f4df2b5c318e89a9eb16d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1730983365967 2024-11-07T12:42:49,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742312_1488 (size=32073) 2024-11-07T12:42:49,259 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f0feb6f84d4477997d169b50d196cdd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1730983367200 2024-11-07T12:42:49,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-07T12:42:49,264 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/cc68a3e0dd274d498844ee92198dbd92 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/cc68a3e0dd274d498844ee92198dbd92 2024-11-07T12:42:49,271 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ffbbe32019fd517ea21eab76e142aa7f/A of ffbbe32019fd517ea21eab76e142aa7f into cc68a3e0dd274d498844ee92198dbd92(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:42:49,271 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffbbe32019fd517ea21eab76e142aa7f: 2024-11-07T12:42:49,271 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., storeName=ffbbe32019fd517ea21eab76e142aa7f/A, priority=12, startTime=1730983368817; duration=0sec 2024-11-07T12:42:49,271 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:42:49,272 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffbbe32019fd517ea21eab76e142aa7f:A 2024-11-07T12:42:49,272 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffbbe32019fd517ea21eab76e142aa7f#C#compaction#410 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:42:49,272 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/68623b16d2964570b8aefc12b98b0c95 is 50, key is test_row_0/C:col10/1730983367877/Put/seqid=0 2024-11-07T12:42:49,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742313_1489 (size=13119) 2024-11-07T12:42:49,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:49,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55828 deadline: 1730983429266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:49,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:49,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:49,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:49,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:49,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55844 deadline: 1730983429271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:49,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:49,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:49,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:49,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:49,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55856 deadline: 1730983429286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:49,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:49,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:49,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:49,380 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/65888d63555a46a69abe47c1e8c64ea0 2024-11-07T12:42:49,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/28a37e937a92412bb189a2b842aff785 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/28a37e937a92412bb189a2b842aff785 2024-11-07T12:42:49,388 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/28a37e937a92412bb189a2b842aff785, entries=150, sequenceid=412, filesize=30.5 K 2024-11-07T12:42:49,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/825328eefa2243fd8974c98aefee7946 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/825328eefa2243fd8974c98aefee7946 2024-11-07T12:42:49,396 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/825328eefa2243fd8974c98aefee7946, entries=150, sequenceid=412, filesize=12.0 K 2024-11-07T12:42:49,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/65888d63555a46a69abe47c1e8c64ea0 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/65888d63555a46a69abe47c1e8c64ea0 2024-11-07T12:42:49,399 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/65888d63555a46a69abe47c1e8c64ea0, entries=150, sequenceid=412, filesize=12.0 K 2024-11-07T12:42:49,400 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for ffbbe32019fd517ea21eab76e142aa7f in 506ms, sequenceid=412, compaction requested=false 2024-11-07T12:42:49,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for ffbbe32019fd517ea21eab76e142aa7f: 2024-11-07T12:42:49,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:49,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-07T12:42:49,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-07T12:42:49,403 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-07T12:42:49,403 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2760 sec 2024-11-07T12:42:49,405 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 1.2810 sec 2024-11-07T12:42:49,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:49,411 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ffbbe32019fd517ea21eab76e142aa7f 3/3 column families, dataSize=154.31 KB heapSize=440.77 KB 2024-11-07T12:42:49,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffbbe32019fd517ea21eab76e142aa7f, store=A 2024-11-07T12:42:49,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-07T12:42:49,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffbbe32019fd517ea21eab76e142aa7f, store=B 2024-11-07T12:42:49,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-07T12:42:49,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffbbe32019fd517ea21eab76e142aa7f, store=C 2024-11-07T12:42:49,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-07T12:42:49,421 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110756e500440fa04922a5254279e2d4f61d_ffbbe32019fd517ea21eab76e142aa7f is 50, key is test_row_0/A:col10/1730983369410/Put/seqid=0 2024-11-07T12:42:49,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742314_1490 (size=20074) 2024-11-07T12:42:49,434 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,437 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110756e500440fa04922a5254279e2d4f61d_ffbbe32019fd517ea21eab76e142aa7f to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110756e500440fa04922a5254279e2d4f61d_ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:49,438 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/d10040eafe1a4aea8c4ecd1e8000a782, store: [table=TestAcidGuarantees family=A region=ffbbe32019fd517ea21eab76e142aa7f] 2024-11-07T12:42:49,438 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/d10040eafe1a4aea8c4ecd1e8000a782 is 175, key is test_row_0/A:col10/1730983369410/Put/seqid=0 2024-11-07T12:42:49,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742315_1491 (size=57333) 2024-11-07T12:42:49,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:49,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55828 deadline: 1730983429431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:49,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:49,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:49,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:49,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:49,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55844 deadline: 1730983429432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:49,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:49,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:49,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:49,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:49,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55856 deadline: 1730983429444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:49,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:49,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:49,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:49,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:49,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55828 deadline: 1730983429562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:49,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:49,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:49,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55844 deadline: 1730983429562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:49,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:49,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:49,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:49,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:49,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:49,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:49,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55856 deadline: 1730983429571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:49,585 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:49,585 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:49,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:49,683 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/68623b16d2964570b8aefc12b98b0c95 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/68623b16d2964570b8aefc12b98b0c95 2024-11-07T12:42:49,688 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ffbbe32019fd517ea21eab76e142aa7f/C of ffbbe32019fd517ea21eab76e142aa7f into 68623b16d2964570b8aefc12b98b0c95(size=12.8 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:42:49,688 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffbbe32019fd517ea21eab76e142aa7f: 2024-11-07T12:42:49,688 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., storeName=ffbbe32019fd517ea21eab76e142aa7f/C, priority=12, startTime=1730983368818; duration=0sec 2024-11-07T12:42:49,688 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:42:49,688 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffbbe32019fd517ea21eab76e142aa7f:C 2024-11-07T12:42:49,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:49,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55828 deadline: 1730983429781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:49,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:49,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:49,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:49,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:49,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55844 deadline: 1730983429781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:49,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:49,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:49,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:49,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:49,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55856 deadline: 1730983429792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:49,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:49,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:49,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:49,843 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=453, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/d10040eafe1a4aea8c4ecd1e8000a782 2024-11-07T12:42:49,861 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/b47e1d2e450e4e3faf04734e9843b063 is 50, key is test_row_0/B:col10/1730983369410/Put/seqid=0 2024-11-07T12:42:49,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742316_1492 (size=12301) 2024-11-07T12:42:49,871 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/b47e1d2e450e4e3faf04734e9843b063 2024-11-07T12:42:49,880 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/c6a872de2fd046a080cf8eb72d8cbb9c is 50, key is test_row_0/C:col10/1730983369410/Put/seqid=0 2024-11-07T12:42:49,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742317_1493 (size=12301) 2024-11-07T12:42:49,884 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/c6a872de2fd046a080cf8eb72d8cbb9c 2024-11-07T12:42:49,890 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/d10040eafe1a4aea8c4ecd1e8000a782 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/d10040eafe1a4aea8c4ecd1e8000a782 2024-11-07T12:42:49,893 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/d10040eafe1a4aea8c4ecd1e8000a782, entries=300, sequenceid=453, filesize=56.0 K 2024-11-07T12:42:49,894 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/b47e1d2e450e4e3faf04734e9843b063 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/b47e1d2e450e4e3faf04734e9843b063 2024-11-07T12:42:49,898 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/b47e1d2e450e4e3faf04734e9843b063, entries=150, sequenceid=453, filesize=12.0 K 2024-11-07T12:42:49,899 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/c6a872de2fd046a080cf8eb72d8cbb9c as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/c6a872de2fd046a080cf8eb72d8cbb9c 2024-11-07T12:42:49,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,902 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/c6a872de2fd046a080cf8eb72d8cbb9c, entries=150, sequenceid=453, filesize=12.0 K 2024-11-07T12:42:49,903 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~440.72 KB/451296, currentSize=26.84 KB/27480 for ffbbe32019fd517ea21eab76e142aa7f in 492ms, sequenceid=453, compaction requested=true 2024-11-07T12:42:49,903 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ffbbe32019fd517ea21eab76e142aa7f: 2024-11-07T12:42:49,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffbbe32019fd517ea21eab76e142aa7f:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:42:49,903 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:42:49,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:42:49,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffbbe32019fd517ea21eab76e142aa7f:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:42:49,903 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:42:49,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:42:49,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffbbe32019fd517ea21eab76e142aa7f:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:42:49,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:42:49,904 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 120661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:42:49,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,904 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): ffbbe32019fd517ea21eab76e142aa7f/A is initiating minor compaction (all files) 2024-11-07T12:42:49,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,904 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffbbe32019fd517ea21eab76e142aa7f/A in TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:49,905 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/cc68a3e0dd274d498844ee92198dbd92, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/28a37e937a92412bb189a2b842aff785, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/d10040eafe1a4aea8c4ecd1e8000a782] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp, totalSize=117.8 K 2024-11-07T12:42:49,905 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:49,905 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. files: [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/cc68a3e0dd274d498844ee92198dbd92, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/28a37e937a92412bb189a2b842aff785, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/d10040eafe1a4aea8c4ecd1e8000a782] 2024-11-07T12:42:49,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,905 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:42:49,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,905 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): ffbbe32019fd517ea21eab76e142aa7f/B is initiating minor compaction (all files) 2024-11-07T12:42:49,906 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffbbe32019fd517ea21eab76e142aa7f/B in TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:49,906 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/b3632077c45442cfa4a6b6a5cc1ab324, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/825328eefa2243fd8974c98aefee7946, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/b47e1d2e450e4e3faf04734e9843b063] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp, totalSize=36.8 K 2024-11-07T12:42:49,906 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc68a3e0dd274d498844ee92198dbd92, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1730983367200 2024-11-07T12:42:49,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,909 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting b3632077c45442cfa4a6b6a5cc1ab324, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1730983367200 2024-11-07T12:42:49,909 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28a37e937a92412bb189a2b842aff785, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1730983367904 2024-11-07T12:42:49,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,909 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 825328eefa2243fd8974c98aefee7946, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1730983367904 2024-11-07T12:42:49,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,909 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting d10040eafe1a4aea8c4ecd1e8000a782, keycount=300, bloomtype=ROW, size=56.0 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1730983369263 2024-11-07T12:42:49,910 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting b47e1d2e450e4e3faf04734e9843b063, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1730983369284 2024-11-07T12:42:49,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,917 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ffbbe32019fd517ea21eab76e142aa7f] 2024-11-07T12:42:49,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,919 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024110736843a1e0d2c496db77c28aad75163cd_ffbbe32019fd517ea21eab76e142aa7f store=[table=TestAcidGuarantees family=A region=ffbbe32019fd517ea21eab76e142aa7f] 2024-11-07T12:42:49,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,926 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffbbe32019fd517ea21eab76e142aa7f#B#compaction#414 average throughput is 0.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:42:49,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,927 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/919b4dd71dfc4e2283da486e448fc3b2 is 50, key is test_row_0/B:col10/1730983369410/Put/seqid=0 2024-11-07T12:42:49,927 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024110736843a1e0d2c496db77c28aad75163cd_ffbbe32019fd517ea21eab76e142aa7f, store=[table=TestAcidGuarantees family=A region=ffbbe32019fd517ea21eab76e142aa7f] 2024-11-07T12:42:49,927 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110736843a1e0d2c496db77c28aad75163cd_ffbbe32019fd517ea21eab76e142aa7f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ffbbe32019fd517ea21eab76e142aa7f] 2024-11-07T12:42:49,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742319_1495 (size=4469) 2024-11-07T12:42:49,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,989 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffbbe32019fd517ea21eab76e142aa7f#A#compaction#415 average throughput is 0.34 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:42:49,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,990 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/39f22b17b5e842f98bba3553f90793d3 is 175, key is test_row_0/A:col10/1730983369410/Put/seqid=0 2024-11-07T12:42:49,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742318_1494 (size=13221) 2024-11-07T12:42:49,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:49,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,003 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/919b4dd71dfc4e2283da486e448fc3b2 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/919b4dd71dfc4e2283da486e448fc3b2 2024-11-07T12:42:50,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,010 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ffbbe32019fd517ea21eab76e142aa7f/B of ffbbe32019fd517ea21eab76e142aa7f into 919b4dd71dfc4e2283da486e448fc3b2(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:42:50,010 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffbbe32019fd517ea21eab76e142aa7f: 2024-11-07T12:42:50,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,010 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., storeName=ffbbe32019fd517ea21eab76e142aa7f/B, priority=13, startTime=1730983369903; duration=0sec 2024-11-07T12:42:50,010 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:42:50,010 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffbbe32019fd517ea21eab76e142aa7f:B 2024-11-07T12:42:50,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,010 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:42:50,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,011 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:42:50,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,011 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): ffbbe32019fd517ea21eab76e142aa7f/C is initiating minor compaction (all files) 2024-11-07T12:42:50,011 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffbbe32019fd517ea21eab76e142aa7f/C in TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:50,011 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/68623b16d2964570b8aefc12b98b0c95, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/65888d63555a46a69abe47c1e8c64ea0, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/c6a872de2fd046a080cf8eb72d8cbb9c] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp, totalSize=36.8 K 2024-11-07T12:42:50,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,012 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 68623b16d2964570b8aefc12b98b0c95, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1730983367200 2024-11-07T12:42:50,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,012 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 65888d63555a46a69abe47c1e8c64ea0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1730983367904 2024-11-07T12:42:50,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,013 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting c6a872de2fd046a080cf8eb72d8cbb9c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1730983369284 2024-11-07T12:42:50,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,031 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffbbe32019fd517ea21eab76e142aa7f#C#compaction#416 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:42:50,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,031 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/3a268dbac01f4048b1a0a6bcba6f35c2 is 50, key is test_row_0/C:col10/1730983369410/Put/seqid=0 2024-11-07T12:42:50,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742320_1496 (size=32175) 2024-11-07T12:42:50,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,038 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/39f22b17b5e842f98bba3553f90793d3 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/39f22b17b5e842f98bba3553f90793d3 2024-11-07T12:42:50,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,044 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ffbbe32019fd517ea21eab76e142aa7f/A of ffbbe32019fd517ea21eab76e142aa7f into 39f22b17b5e842f98bba3553f90793d3(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:42:50,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,044 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffbbe32019fd517ea21eab76e142aa7f: 2024-11-07T12:42:50,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,044 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., storeName=ffbbe32019fd517ea21eab76e142aa7f/A, priority=13, startTime=1730983369903; duration=0sec 2024-11-07T12:42:50,044 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:42:50,044 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffbbe32019fd517ea21eab76e142aa7f:A 2024-11-07T12:42:50,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742321_1497 (size=13221) 2024-11-07T12:42:50,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,072 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/3a268dbac01f4048b1a0a6bcba6f35c2 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/3a268dbac01f4048b1a0a6bcba6f35c2 2024-11-07T12:42:50,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,077 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ffbbe32019fd517ea21eab76e142aa7f/C of ffbbe32019fd517ea21eab76e142aa7f into 3a268dbac01f4048b1a0a6bcba6f35c2(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:42:50,077 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffbbe32019fd517ea21eab76e142aa7f: 2024-11-07T12:42:50,077 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., storeName=ffbbe32019fd517ea21eab76e142aa7f/C, priority=13, startTime=1730983369903; duration=0sec 2024-11-07T12:42:50,077 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:42:50,077 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffbbe32019fd517ea21eab76e142aa7f:C 2024-11-07T12:42:50,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,137 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ffbbe32019fd517ea21eab76e142aa7f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-07T12:42:50,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:50,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffbbe32019fd517ea21eab76e142aa7f, store=A 2024-11-07T12:42:50,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:50,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffbbe32019fd517ea21eab76e142aa7f, store=B 2024-11-07T12:42:50,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:50,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffbbe32019fd517ea21eab76e142aa7f, store=C 2024-11-07T12:42:50,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:50,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,152 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107adec7d4aa1d04f3eac8667083e621085_ffbbe32019fd517ea21eab76e142aa7f is 50, key is test_row_0/A:col10/1730983369430/Put/seqid=0 2024-11-07T12:42:50,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742322_1498 (size=17534) 2024-11-07T12:42:50,211 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,214 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107adec7d4aa1d04f3eac8667083e621085_ffbbe32019fd517ea21eab76e142aa7f to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107adec7d4aa1d04f3eac8667083e621085_ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:50,215 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/05524e9858b241f090d08b67efbba432, store: [table=TestAcidGuarantees family=A region=ffbbe32019fd517ea21eab76e142aa7f] 2024-11-07T12:42:50,216 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/05524e9858b241f090d08b67efbba432 is 175, key is test_row_0/A:col10/1730983369430/Put/seqid=0 2024-11-07T12:42:50,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742323_1499 (size=48639) 2024-11-07T12:42:50,251 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=467, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/05524e9858b241f090d08b67efbba432 2024-11-07T12:42:50,260 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1730983370222, max timestamp=1730983370254Num uniques 859; 2024-11-07T12:42:50,260 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1730983370222, max timestamp=1730983370254Num uniques 859; 2024-11-07T12:42:50,260 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1730983370222, max timestamp=1730983370254Num uniques 859; 2024-11-07T12:42:50,260 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-07T12:42:50,260 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-07T12:42:50,260 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-07T12:42:50,262 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/83625c98d9e049b38899a3d5ba6dd9ec is 50, key is test_row_0/B:col10/1730983369430/Put/seqid=0 2024-11-07T12:42:50,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-07T12:42:50,274 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:42:50,275 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:42:50,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-07T12:42:50,279 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:42:50,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-07T12:42:50,280 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:42:50,280 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:42:50,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742324_1500 (size=12301) 2024-11-07T12:42:50,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:50,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55844 deadline: 1730983430326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:50,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:50,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:50,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:50,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:50,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55828 deadline: 1730983430327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:50,346 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:50,346 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:50,346 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:50,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:50,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55856 deadline: 1730983430336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:50,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:50,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:50,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:50,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-07T12:42:50,440 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-07T12:42:50,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:50,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. as already flushing 2024-11-07T12:42:50,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:50,441 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:50,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:50,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:50,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:50,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55844 deadline: 1730983430452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:50,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:50,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55828 deadline: 1730983430452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:50,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:50,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:50,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:50,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:50,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:50,473 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:50,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:50,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55856 deadline: 1730983430464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:50,481 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:50,481 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:50,481 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:50,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-07T12:42:50,595 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-07T12:42:50,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:50,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. as already flushing 2024-11-07T12:42:50,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:50,595 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:50,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:50,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:50,688 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=467 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/83625c98d9e049b38899a3d5ba6dd9ec 2024-11-07T12:42:50,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:50,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55844 deadline: 1730983430685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:50,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:50,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:50,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:50,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:50,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55828 deadline: 1730983430685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:50,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:50,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:50,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:50,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:50,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55856 deadline: 1730983430692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:50,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:50,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:50,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:50,713 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/eeba5c44a9c94ac6b5d91801471302a7 is 50, key is test_row_0/C:col10/1730983369430/Put/seqid=0 2024-11-07T12:42:50,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742325_1501 (size=12301) 2024-11-07T12:42:50,725 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=467 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/eeba5c44a9c94ac6b5d91801471302a7 2024-11-07T12:42:50,731 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/05524e9858b241f090d08b67efbba432 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/05524e9858b241f090d08b67efbba432 2024-11-07T12:42:50,735 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/05524e9858b241f090d08b67efbba432, entries=250, sequenceid=467, filesize=47.5 K 2024-11-07T12:42:50,735 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/83625c98d9e049b38899a3d5ba6dd9ec as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/83625c98d9e049b38899a3d5ba6dd9ec 2024-11-07T12:42:50,742 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/83625c98d9e049b38899a3d5ba6dd9ec, entries=150, sequenceid=467, filesize=12.0 K 2024-11-07T12:42:50,743 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/eeba5c44a9c94ac6b5d91801471302a7 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/eeba5c44a9c94ac6b5d91801471302a7 2024-11-07T12:42:50,749 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/eeba5c44a9c94ac6b5d91801471302a7, entries=150, sequenceid=467, filesize=12.0 K 2024-11-07T12:42:50,750 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for ffbbe32019fd517ea21eab76e142aa7f in 613ms, sequenceid=467, compaction requested=false 2024-11-07T12:42:50,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ffbbe32019fd517ea21eab76e142aa7f: 2024-11-07T12:42:50,750 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-07T12:42:50,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:50,750 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing ffbbe32019fd517ea21eab76e142aa7f 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-07T12:42:50,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffbbe32019fd517ea21eab76e142aa7f, store=A 2024-11-07T12:42:50,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:42:50,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffbbe32019fd517ea21eab76e142aa7f, store=B 2024-11-07T12:42:50,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:42:50,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffbbe32019fd517ea21eab76e142aa7f, store=C 2024-11-07T12:42:50,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:42:50,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107c4094e9929794487baa74ce1c000d337_ffbbe32019fd517ea21eab76e142aa7f is 50, key is test_row_0/A:col10/1730983370326/Put/seqid=0 2024-11-07T12:42:50,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742326_1502 (size=12454) 2024-11-07T12:42:50,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:50,801 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107c4094e9929794487baa74ce1c000d337_ffbbe32019fd517ea21eab76e142aa7f to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107c4094e9929794487baa74ce1c000d337_ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:50,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/36e7b9626ff04db3a64c5689a23b06c7, store: [table=TestAcidGuarantees family=A region=ffbbe32019fd517ea21eab76e142aa7f] 2024-11-07T12:42:50,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/36e7b9626ff04db3a64c5689a23b06c7 is 175, key is test_row_0/A:col10/1730983370326/Put/seqid=0 2024-11-07T12:42:50,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742327_1503 (size=31255) 2024-11-07T12:42:50,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:50,822 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. as already flushing 2024-11-07T12:42:50,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:50,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55862 deadline: 1730983430874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:50,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:50,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55858 deadline: 1730983430874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:50,877 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:50,877 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:50,878 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:50,879 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:50,879 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:50,879 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:50,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-07T12:42:50,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:50,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55862 deadline: 1730983430981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:50,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:50,990 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:50,990 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:50,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55858 deadline: 1730983430982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:50,990 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:50,990 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:50,990 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:50,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:51,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:51,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55844 deadline: 1730983431002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:51,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:51,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:51,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:51,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:51,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55828 deadline: 1730983431002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:51,014 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:51,014 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:51,014 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:51,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:51,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55856 deadline: 1730983431011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:51,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:51,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:51,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:51,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:51,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55862 deadline: 1730983431192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:51,199 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:51,199 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:51,199 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:51,210 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=503, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/36e7b9626ff04db3a64c5689a23b06c7 2024-11-07T12:42:51,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:51,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55858 deadline: 1730983431201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:51,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/d2fed702c6e944abb64a713697d1b1e0 is 50, key is test_row_0/B:col10/1730983370326/Put/seqid=0 2024-11-07T12:42:51,219 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:51,219 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:51,219 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:51,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742328_1504 (size=12301) 2024-11-07T12:42:51,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-07T12:42:51,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:51,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55862 deadline: 1730983431501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:51,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:51,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:51,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:51,530 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:51,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55844 deadline: 1730983431522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:51,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:51,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:51,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55828 deadline: 1730983431522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:51,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:51,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:51,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:51,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55858 deadline: 1730983431522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:51,531 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:51,532 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:51,532 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:51,532 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:51,532 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:51,532 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:51,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:51,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55856 deadline: 1730983431531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:51,544 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:51,544 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:51,544 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:51,629 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=503 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/d2fed702c6e944abb64a713697d1b1e0 2024-11-07T12:42:51,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/c56871bf91144dfabfb4be0508434c6f is 50, key is test_row_0/C:col10/1730983370326/Put/seqid=0 2024-11-07T12:42:51,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742329_1505 (size=12301) 2024-11-07T12:42:52,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:52,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55862 deadline: 1730983432013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:52,021 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:52,021 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:52,021 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:52,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:52,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55858 deadline: 1730983432041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:52,044 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:52,044 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:52,045 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:52,047 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=503 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/c56871bf91144dfabfb4be0508434c6f 2024-11-07T12:42:52,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/36e7b9626ff04db3a64c5689a23b06c7 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/36e7b9626ff04db3a64c5689a23b06c7 2024-11-07T12:42:52,056 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/36e7b9626ff04db3a64c5689a23b06c7, entries=150, sequenceid=503, filesize=30.5 K 2024-11-07T12:42:52,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/d2fed702c6e944abb64a713697d1b1e0 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/d2fed702c6e944abb64a713697d1b1e0 2024-11-07T12:42:52,064 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/d2fed702c6e944abb64a713697d1b1e0, entries=150, sequenceid=503, filesize=12.0 K 2024-11-07T12:42:52,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/c56871bf91144dfabfb4be0508434c6f as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/c56871bf91144dfabfb4be0508434c6f 2024-11-07T12:42:52,068 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/c56871bf91144dfabfb4be0508434c6f, entries=150, sequenceid=503, filesize=12.0 K 2024-11-07T12:42:52,069 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for ffbbe32019fd517ea21eab76e142aa7f in 1319ms, sequenceid=503, compaction requested=true 2024-11-07T12:42:52,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for ffbbe32019fd517ea21eab76e142aa7f: 2024-11-07T12:42:52,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:52,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-07T12:42:52,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-07T12:42:52,073 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-07T12:42:52,073 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7910 sec 2024-11-07T12:42:52,075 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 1.7990 sec 2024-11-07T12:42:52,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-07T12:42:52,422 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:42:52,422 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:42:52,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-07T12:42:52,425 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:42:52,426 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:42:52,426 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:42:52,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-07T12:42:52,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-07T12:42:52,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:52,545 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ffbbe32019fd517ea21eab76e142aa7f 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-07T12:42:52,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffbbe32019fd517ea21eab76e142aa7f, store=A 2024-11-07T12:42:52,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:52,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffbbe32019fd517ea21eab76e142aa7f, store=B 2024-11-07T12:42:52,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:52,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffbbe32019fd517ea21eab76e142aa7f, store=C 2024-11-07T12:42:52,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:52,566 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411078c294f2cde414b478bfb3b8b3822497f_ffbbe32019fd517ea21eab76e142aa7f is 50, key is test_row_0/A:col10/1730983370871/Put/seqid=0 2024-11-07T12:42:52,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742330_1506 (size=14994) 2024-11-07T12:42:52,580 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-07T12:42:52,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:52,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. as already flushing 2024-11-07T12:42:52,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:52,581 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:52,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:52,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:52,624 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1730983372582, max timestamp=1730983372620Num uniques 859; 2024-11-07T12:42:52,624 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-07T12:42:52,624 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1730983372582, max timestamp=1730983372620Num uniques 859; 2024-11-07T12:42:52,624 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-07T12:42:52,626 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1730983372582, max timestamp=1730983372620Num uniques 859; 2024-11-07T12:42:52,626 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-07T12:42:52,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:52,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55856 deadline: 1730983432687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:52,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:52,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:52,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:52,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:52,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55844 deadline: 1730983432702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:52,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:52,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:52,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55828 deadline: 1730983432703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:52,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:52,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:52,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:52,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:52,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:52,734 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-07T12:42:52,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:52,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. as already flushing 2024-11-07T12:42:52,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:52,735 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:52,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:52,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:52,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-07T12:42:52,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:52,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55828 deadline: 1730983432812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:52,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:52,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:52,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:52,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:52,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55856 deadline: 1730983432812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:52,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:52,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55844 deadline: 1730983432812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:52,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:52,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:52,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:52,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:52,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:52,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:52,888 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-07T12:42:52,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:52,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. as already flushing 2024-11-07T12:42:52,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:52,888 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:52,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:52,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:52,979 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:52,983 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411078c294f2cde414b478bfb3b8b3822497f_ffbbe32019fd517ea21eab76e142aa7f to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411078c294f2cde414b478bfb3b8b3822497f_ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:52,984 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/5c56abc2f5c04f05925b72798ba90f69, store: [table=TestAcidGuarantees family=A region=ffbbe32019fd517ea21eab76e142aa7f] 2024-11-07T12:42:52,985 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/5c56abc2f5c04f05925b72798ba90f69 is 175, key is test_row_0/A:col10/1730983370871/Put/seqid=0 2024-11-07T12:42:53,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742331_1507 (size=39949) 2024-11-07T12:42:53,004 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=514, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/5c56abc2f5c04f05925b72798ba90f69 2024-11-07T12:42:53,013 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/8b4c087d29bf4b4493695f02ac8a1bb6 is 50, key is test_row_0/B:col10/1730983370871/Put/seqid=0 2024-11-07T12:42:53,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742332_1508 (size=12301) 2024-11-07T12:42:53,023 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=514 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/8b4c087d29bf4b4493695f02ac8a1bb6 2024-11-07T12:42:53,033 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/611d6e99a7914503bb979433722e1659 is 50, key is test_row_0/C:col10/1730983370871/Put/seqid=0 2024-11-07T12:42:53,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:53,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55828 deadline: 1730983433032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:53,040 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:53,040 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:53,040 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:53,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:53,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55856 deadline: 1730983433032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:53,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:53,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55844 deadline: 1730983433032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:53,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:53,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:53,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:53,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:53,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55862 deadline: 1730983433033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:53,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:53,042 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to db9ad1cb6cf9,34293,1730983260217 for 'test_row_2' in ffbbe32019fd517ea21eab76e142aa7f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:42:53,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:53,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:53,043 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:53,043 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:53,043 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:53,044 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-07T12:42:53,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:53,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. as already flushing 2024-11-07T12:42:53,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:53,044 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:53,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:53,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:42:53,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-07T12:42:53,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742333_1509 (size=12301) 2024-11-07T12:42:53,062 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=514 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/611d6e99a7914503bb979433722e1659 2024-11-07T12:42:53,075 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/5c56abc2f5c04f05925b72798ba90f69 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/5c56abc2f5c04f05925b72798ba90f69 2024-11-07T12:42:53,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:53,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55858 deadline: 1730983433062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:53,078 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to db9ad1cb6cf9,34293,1730983260217 for 'test_row_0' in ffbbe32019fd517ea21eab76e142aa7f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:42:53,078 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:53,079 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:53,079 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:53,081 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/5c56abc2f5c04f05925b72798ba90f69, entries=200, sequenceid=514, filesize=39.0 K 2024-11-07T12:42:53,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/8b4c087d29bf4b4493695f02ac8a1bb6 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/8b4c087d29bf4b4493695f02ac8a1bb6 2024-11-07T12:42:53,097 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/8b4c087d29bf4b4493695f02ac8a1bb6, entries=150, sequenceid=514, filesize=12.0 K 2024-11-07T12:42:53,098 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/611d6e99a7914503bb979433722e1659 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/611d6e99a7914503bb979433722e1659 2024-11-07T12:42:53,102 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/611d6e99a7914503bb979433722e1659, entries=150, sequenceid=514, filesize=12.0 K 2024-11-07T12:42:53,106 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for ffbbe32019fd517ea21eab76e142aa7f in 562ms, sequenceid=514, compaction requested=true 2024-11-07T12:42:53,106 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ffbbe32019fd517ea21eab76e142aa7f: 2024-11-07T12:42:53,106 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:42:53,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffbbe32019fd517ea21eab76e142aa7f:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:42:53,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:42:53,106 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:42:53,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffbbe32019fd517ea21eab76e142aa7f:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:42:53,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:42:53,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffbbe32019fd517ea21eab76e142aa7f:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:42:53,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:42:53,107 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 152018 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:42:53,107 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): ffbbe32019fd517ea21eab76e142aa7f/A is initiating minor compaction (all files) 2024-11-07T12:42:53,108 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffbbe32019fd517ea21eab76e142aa7f/A in TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:53,108 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/39f22b17b5e842f98bba3553f90793d3, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/05524e9858b241f090d08b67efbba432, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/36e7b9626ff04db3a64c5689a23b06c7, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/5c56abc2f5c04f05925b72798ba90f69] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp, totalSize=148.5 K 2024-11-07T12:42:53,108 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:53,108 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. files: [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/39f22b17b5e842f98bba3553f90793d3, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/05524e9858b241f090d08b67efbba432, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/36e7b9626ff04db3a64c5689a23b06c7, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/5c56abc2f5c04f05925b72798ba90f69] 2024-11-07T12:42:53,108 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50124 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:42:53,108 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): ffbbe32019fd517ea21eab76e142aa7f/B is initiating minor compaction (all files) 2024-11-07T12:42:53,108 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffbbe32019fd517ea21eab76e142aa7f/B in TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:53,108 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/919b4dd71dfc4e2283da486e448fc3b2, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/83625c98d9e049b38899a3d5ba6dd9ec, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/d2fed702c6e944abb64a713697d1b1e0, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/8b4c087d29bf4b4493695f02ac8a1bb6] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp, totalSize=48.9 K 2024-11-07T12:42:53,109 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 39f22b17b5e842f98bba3553f90793d3, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1730983369284 2024-11-07T12:42:53,109 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 919b4dd71dfc4e2283da486e448fc3b2, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1730983369284 2024-11-07T12:42:53,109 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 83625c98d9e049b38899a3d5ba6dd9ec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=467, earliestPutTs=1730983369430 2024-11-07T12:42:53,109 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05524e9858b241f090d08b67efbba432, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=467, earliestPutTs=1730983369430 2024-11-07T12:42:53,112 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36e7b9626ff04db3a64c5689a23b06c7, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=503, earliestPutTs=1730983370306 2024-11-07T12:42:53,113 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting d2fed702c6e944abb64a713697d1b1e0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=503, earliestPutTs=1730983370306 2024-11-07T12:42:53,113 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c56abc2f5c04f05925b72798ba90f69, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=514, earliestPutTs=1730983370821 2024-11-07T12:42:53,113 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b4c087d29bf4b4493695f02ac8a1bb6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=514, earliestPutTs=1730983370821 2024-11-07T12:42:53,140 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffbbe32019fd517ea21eab76e142aa7f#B#compaction#426 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:42:53,141 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/b1ecdda4a2f147c988c295e917688a4d is 50, key is test_row_0/B:col10/1730983370871/Put/seqid=0 2024-11-07T12:42:53,154 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ffbbe32019fd517ea21eab76e142aa7f] 2024-11-07T12:42:53,201 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-07T12:42:53,202 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411078aa53b4a4d2144d997fc69e85f7336c9_ffbbe32019fd517ea21eab76e142aa7f store=[table=TestAcidGuarantees family=A region=ffbbe32019fd517ea21eab76e142aa7f] 2024-11-07T12:42:53,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:53,205 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing ffbbe32019fd517ea21eab76e142aa7f 3/3 column families, dataSize=134.18 KB heapSize=370.92 KB 2024-11-07T12:42:53,205 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411078aa53b4a4d2144d997fc69e85f7336c9_ffbbe32019fd517ea21eab76e142aa7f, store=[table=TestAcidGuarantees family=A region=ffbbe32019fd517ea21eab76e142aa7f] 2024-11-07T12:42:53,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffbbe32019fd517ea21eab76e142aa7f, store=A 2024-11-07T12:42:53,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:42:53,205 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411078aa53b4a4d2144d997fc69e85f7336c9_ffbbe32019fd517ea21eab76e142aa7f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ffbbe32019fd517ea21eab76e142aa7f] 2024-11-07T12:42:53,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffbbe32019fd517ea21eab76e142aa7f, store=B 2024-11-07T12:42:53,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:42:53,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffbbe32019fd517ea21eab76e142aa7f, store=C 2024-11-07T12:42:53,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:42:53,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742334_1510 (size=13357) 2024-11-07T12:42:53,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742335_1511 (size=4469) 2024-11-07T12:42:53,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107309c9464ab5b400c858d926330f156df_ffbbe32019fd517ea21eab76e142aa7f is 50, key is test_row_0/A:col10/1730983372700/Put/seqid=0 2024-11-07T12:42:53,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742336_1512 (size=12454) 2024-11-07T12:42:53,366 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. as already flushing 2024-11-07T12:42:53,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:53,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:53,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55856 deadline: 1730983433402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:53,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:53,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:53,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:53,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:53,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55844 deadline: 1730983433419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:53,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:53,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:53,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:53,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:53,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55828 deadline: 1730983433429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:53,445 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:53,445 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:53,445 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:53,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:53,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55856 deadline: 1730983433532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:53,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:53,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:53,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:53,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:53,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55844 deadline: 1730983433541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:53,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:53,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:53,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:53,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-07T12:42:53,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:53,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55828 deadline: 1730983433551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:53,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:53,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:53,567 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:53,625 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/b1ecdda4a2f147c988c295e917688a4d as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/b1ecdda4a2f147c988c295e917688a4d 2024-11-07T12:42:53,630 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ffbbe32019fd517ea21eab76e142aa7f/B of ffbbe32019fd517ea21eab76e142aa7f into b1ecdda4a2f147c988c295e917688a4d(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:42:53,630 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffbbe32019fd517ea21eab76e142aa7f: 2024-11-07T12:42:53,630 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., storeName=ffbbe32019fd517ea21eab76e142aa7f/B, priority=12, startTime=1730983373106; duration=0sec 2024-11-07T12:42:53,630 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:42:53,630 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffbbe32019fd517ea21eab76e142aa7f:B 2024-11-07T12:42:53,630 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:42:53,631 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50124 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:42:53,631 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): ffbbe32019fd517ea21eab76e142aa7f/C is initiating minor compaction (all files) 2024-11-07T12:42:53,631 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffbbe32019fd517ea21eab76e142aa7f/C in TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:53,631 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/3a268dbac01f4048b1a0a6bcba6f35c2, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/eeba5c44a9c94ac6b5d91801471302a7, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/c56871bf91144dfabfb4be0508434c6f, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/611d6e99a7914503bb979433722e1659] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp, totalSize=48.9 K 2024-11-07T12:42:53,632 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a268dbac01f4048b1a0a6bcba6f35c2, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1730983369284 2024-11-07T12:42:53,632 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting eeba5c44a9c94ac6b5d91801471302a7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=467, earliestPutTs=1730983369430 2024-11-07T12:42:53,632 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting c56871bf91144dfabfb4be0508434c6f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=503, earliestPutTs=1730983370306 2024-11-07T12:42:53,633 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 611d6e99a7914503bb979433722e1659, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=514, earliestPutTs=1730983370821 2024-11-07T12:42:53,638 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffbbe32019fd517ea21eab76e142aa7f#A#compaction#427 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:42:53,639 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/a1e7f09337d34167b98cd630679f73d3 is 175, key is test_row_0/A:col10/1730983370871/Put/seqid=0 2024-11-07T12:42:53,649 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffbbe32019fd517ea21eab76e142aa7f#C#compaction#429 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:42:53,650 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/d51c64bc099744148132fef5eaa2fb4c is 50, key is test_row_0/C:col10/1730983370871/Put/seqid=0 2024-11-07T12:42:53,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742337_1513 (size=32311) 2024-11-07T12:42:53,719 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/a1e7f09337d34167b98cd630679f73d3 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/a1e7f09337d34167b98cd630679f73d3 2024-11-07T12:42:53,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742338_1514 (size=13357) 2024-11-07T12:42:53,724 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ffbbe32019fd517ea21eab76e142aa7f/A of ffbbe32019fd517ea21eab76e142aa7f into a1e7f09337d34167b98cd630679f73d3(size=31.6 K), total size for store is 31.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:42:53,724 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffbbe32019fd517ea21eab76e142aa7f: 2024-11-07T12:42:53,724 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., storeName=ffbbe32019fd517ea21eab76e142aa7f/A, priority=12, startTime=1730983373106; duration=0sec 2024-11-07T12:42:53,724 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:42:53,724 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffbbe32019fd517ea21eab76e142aa7f:A 2024-11-07T12:42:53,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:53,729 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107309c9464ab5b400c858d926330f156df_ffbbe32019fd517ea21eab76e142aa7f to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107309c9464ab5b400c858d926330f156df_ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:53,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/21cd23dc124045c49fa50fe07c4ea205, store: [table=TestAcidGuarantees family=A region=ffbbe32019fd517ea21eab76e142aa7f] 2024-11-07T12:42:53,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/21cd23dc124045c49fa50fe07c4ea205 is 175, key is test_row_0/A:col10/1730983372700/Put/seqid=0 2024-11-07T12:42:53,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:53,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55856 deadline: 1730983433751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:53,767 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:53,767 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:53,767 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:53,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742339_1515 (size=31255) 2024-11-07T12:42:53,777 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=548, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/21cd23dc124045c49fa50fe07c4ea205 2024-11-07T12:42:53,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:53,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55844 deadline: 1730983433761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:53,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:53,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:53,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:53,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:53,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55828 deadline: 1730983433772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:53,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:53,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:53,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:53,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/a9f8c6189d844af1a84c0f14c1eebb2c is 50, key is test_row_0/B:col10/1730983372700/Put/seqid=0 2024-11-07T12:42:53,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742340_1516 (size=12301) 2024-11-07T12:42:54,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:54,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55856 deadline: 1730983434074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:54,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:54,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:54,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:54,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:54,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55844 deadline: 1730983434093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:54,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:42:54,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55828 deadline: 1730983434093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:54,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:54,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:54,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:54,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:54,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:42:54,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:42:54,113 INFO [Thread-1741 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1741. 2024-11-07T12:42:54,113 DEBUG [Thread-1741 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:42:54,114 DEBUG [Thread-1741 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:42:54,114 DEBUG [Thread-1741 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:42:54,114 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:42:54,118 INFO [Thread-1739 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1739. 2024-11-07T12:42:54,118 DEBUG [Thread-1739 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:42:54,118 DEBUG [Thread-1739 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:42:54,118 DEBUG [Thread-1739 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:42:54,118 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:42:54,119 INFO [Thread-1745 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1745. 2024-11-07T12:42:54,119 DEBUG [Thread-1745 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:42:54,119 DEBUG [Thread-1745 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:42:54,119 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:42:54,120 DEBUG [Thread-1745 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:42:54,130 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/d51c64bc099744148132fef5eaa2fb4c as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/d51c64bc099744148132fef5eaa2fb4c 2024-11-07T12:42:54,131 INFO [Thread-1743 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1743. 2024-11-07T12:42:54,131 DEBUG [Thread-1743 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:42:54,131 DEBUG [Thread-1743 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:42:54,132 DEBUG [Thread-1743 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:42:54,132 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:42:54,132 INFO [Thread-1747 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1747. 2024-11-07T12:42:54,132 DEBUG [Thread-1747 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:42:54,132 DEBUG [Thread-1747 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:42:54,132 DEBUG [Thread-1747 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:42:54,132 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:42:54,135 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ffbbe32019fd517ea21eab76e142aa7f/C of ffbbe32019fd517ea21eab76e142aa7f into d51c64bc099744148132fef5eaa2fb4c(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:42:54,135 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffbbe32019fd517ea21eab76e142aa7f: 2024-11-07T12:42:54,135 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., storeName=ffbbe32019fd517ea21eab76e142aa7f/C, priority=12, startTime=1730983373107; duration=0sec 2024-11-07T12:42:54,135 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:42:54,135 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffbbe32019fd517ea21eab76e142aa7f:C 2024-11-07T12:42:54,266 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=548 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/a9f8c6189d844af1a84c0f14c1eebb2c 2024-11-07T12:42:54,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/582efd5e85764639a97b2ac9328ce0c2 is 50, key is test_row_0/C:col10/1730983372700/Put/seqid=0 2024-11-07T12:42:54,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742341_1517 (size=12301) 2024-11-07T12:42:54,277 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=548 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/582efd5e85764639a97b2ac9328ce0c2 2024-11-07T12:42:54,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/21cd23dc124045c49fa50fe07c4ea205 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/21cd23dc124045c49fa50fe07c4ea205 2024-11-07T12:42:54,283 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/21cd23dc124045c49fa50fe07c4ea205, entries=150, sequenceid=548, filesize=30.5 K 2024-11-07T12:42:54,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/a9f8c6189d844af1a84c0f14c1eebb2c as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/a9f8c6189d844af1a84c0f14c1eebb2c 2024-11-07T12:42:54,287 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/a9f8c6189d844af1a84c0f14c1eebb2c, entries=150, sequenceid=548, filesize=12.0 K 2024-11-07T12:42:54,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/582efd5e85764639a97b2ac9328ce0c2 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/582efd5e85764639a97b2ac9328ce0c2 2024-11-07T12:42:54,290 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/582efd5e85764639a97b2ac9328ce0c2, entries=150, sequenceid=548, filesize=12.0 K 2024-11-07T12:42:54,291 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~370.88 KB/379776, currentSize=53.67 KB/54960 for ffbbe32019fd517ea21eab76e142aa7f in 1086ms, sequenceid=548, compaction requested=false 2024-11-07T12:42:54,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for ffbbe32019fd517ea21eab76e142aa7f: 2024-11-07T12:42:54,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:54,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-07T12:42:54,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-07T12:42:54,294 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-07T12:42:54,294 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8660 sec 2024-11-07T12:42:54,295 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 1.8710 sec 2024-11-07T12:42:54,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-07T12:42:54,572 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:42:54,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:54,603 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ffbbe32019fd517ea21eab76e142aa7f 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-07T12:42:54,603 INFO [Thread-1734 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1734. 2024-11-07T12:42:54,603 DEBUG [Thread-1734 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:42:54,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffbbe32019fd517ea21eab76e142aa7f, store=A 2024-11-07T12:42:54,603 DEBUG [Thread-1734 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:42:54,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:54,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffbbe32019fd517ea21eab76e142aa7f, store=B 2024-11-07T12:42:54,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:54,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffbbe32019fd517ea21eab76e142aa7f, store=C 2024-11-07T12:42:54,603 DEBUG [Thread-1734 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:42:54,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:54,603 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:42:54,608 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110734c775a436134ce282a69d37dc365409_ffbbe32019fd517ea21eab76e142aa7f is 50, key is test_row_1/A:col10/1730983374601/Put/seqid=0 2024-11-07T12:42:54,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742342_1518 (size=9914) 2024-11-07T12:42:54,612 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:54,614 INFO [Thread-1736 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1736. 2024-11-07T12:42:54,614 INFO [Thread-1730 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1730. 2024-11-07T12:42:54,614 DEBUG [Thread-1736 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:42:54,614 DEBUG [Thread-1736 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:42:54,614 DEBUG [Thread-1730 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:42:54,614 DEBUG [Thread-1730 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:42:54,614 DEBUG [Thread-1736 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:42:54,614 DEBUG [Thread-1730 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:42:54,614 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:42:54,614 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:42:54,616 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110734c775a436134ce282a69d37dc365409_ffbbe32019fd517ea21eab76e142aa7f to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110734c775a436134ce282a69d37dc365409_ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:54,617 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/b4ac82616b1d455fa4cdbc7025ebb8b5, store: [table=TestAcidGuarantees family=A region=ffbbe32019fd517ea21eab76e142aa7f] 2024-11-07T12:42:54,617 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/b4ac82616b1d455fa4cdbc7025ebb8b5 is 175, key is test_row_1/A:col10/1730983374601/Put/seqid=0 2024-11-07T12:42:54,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742343_1519 (size=22561) 2024-11-07T12:42:55,021 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=563, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/b4ac82616b1d455fa4cdbc7025ebb8b5 2024-11-07T12:42:55,027 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/dbaaec1a2ab24e539131e8d12003397e is 50, key is test_row_1/B:col10/1730983374601/Put/seqid=0 2024-11-07T12:42:55,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742344_1520 (size=9857) 2024-11-07T12:42:55,063 INFO [Thread-1732 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1732. 2024-11-07T12:42:55,064 DEBUG [Thread-1732 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:42:55,064 DEBUG [Thread-1732 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:42:55,064 DEBUG [Thread-1732 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:42:55,064 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:42:55,084 INFO [Thread-1728 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1728. 2024-11-07T12:42:55,084 DEBUG [Thread-1728 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:42:55,084 DEBUG [Thread-1728 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:42:55,084 DEBUG [Thread-1728 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:42:55,084 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:42:55,084 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-07T12:42:55,084 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 66 2024-11-07T12:42:55,084 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 116 2024-11-07T12:42:55,084 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-11-07T12:42:55,084 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 105 2024-11-07T12:42:55,084 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 117 2024-11-07T12:42:55,084 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-07T12:42:55,084 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-07T12:42:55,084 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 756 2024-11-07T12:42:55,084 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2263 rows 2024-11-07T12:42:55,084 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 757 2024-11-07T12:42:55,084 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2262 rows 2024-11-07T12:42:55,084 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 759 2024-11-07T12:42:55,084 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2271 rows 2024-11-07T12:42:55,084 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 755 2024-11-07T12:42:55,084 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2265 rows 2024-11-07T12:42:55,084 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 759 2024-11-07T12:42:55,084 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2270 rows 2024-11-07T12:42:55,084 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-07T12:42:55,085 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T12:42:55,085 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:42:55,085 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:42:55,085 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:42:55,086 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-07T12:42:55,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-07T12:42:55,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-07T12:42:55,088 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730983375088"}]},"ts":"1730983375088"} 2024-11-07T12:42:55,090 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-07T12:42:55,090 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-07T12:42:55,090 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-07T12:42:55,092 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ffbbe32019fd517ea21eab76e142aa7f, UNASSIGN}] 2024-11-07T12:42:55,093 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ffbbe32019fd517ea21eab76e142aa7f, UNASSIGN 2024-11-07T12:42:55,093 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=121 updating hbase:meta row=ffbbe32019fd517ea21eab76e142aa7f, regionState=CLOSING, regionLocation=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:42:55,094 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ffbbe32019fd517ea21eab76e142aa7f, UNASSIGN because future has completed 2024-11-07T12:42:55,095 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-07T12:42:55,095 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; CloseRegionProcedure ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217}] 2024-11-07T12:42:55,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-07T12:42:55,247 INFO [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] handler.UnassignRegionHandler(122): Close ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:55,247 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-07T12:42:55,247 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1722): Closing ffbbe32019fd517ea21eab76e142aa7f, disabling compactions & flushes 2024-11-07T12:42:55,247 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:55,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-07T12:42:55,430 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=563 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/dbaaec1a2ab24e539131e8d12003397e 2024-11-07T12:42:55,436 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/bf9207e8495b4ce69a76c48f8b56589a is 50, key is test_row_1/C:col10/1730983374601/Put/seqid=0 2024-11-07T12:42:55,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742345_1521 (size=9857) 2024-11-07T12:42:55,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-07T12:42:55,840 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=563 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/bf9207e8495b4ce69a76c48f8b56589a 2024-11-07T12:42:55,844 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/b4ac82616b1d455fa4cdbc7025ebb8b5 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/b4ac82616b1d455fa4cdbc7025ebb8b5 2024-11-07T12:42:55,847 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/b4ac82616b1d455fa4cdbc7025ebb8b5, entries=100, sequenceid=563, filesize=22.0 K 2024-11-07T12:42:55,848 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/dbaaec1a2ab24e539131e8d12003397e as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/dbaaec1a2ab24e539131e8d12003397e 2024-11-07T12:42:55,852 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/dbaaec1a2ab24e539131e8d12003397e, entries=100, sequenceid=563, filesize=9.6 K 2024-11-07T12:42:55,853 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/bf9207e8495b4ce69a76c48f8b56589a as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/bf9207e8495b4ce69a76c48f8b56589a 2024-11-07T12:42:55,856 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/bf9207e8495b4ce69a76c48f8b56589a, entries=100, sequenceid=563, filesize=9.6 K 2024-11-07T12:42:55,857 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=26.84 KB/27480 for ffbbe32019fd517ea21eab76e142aa7f in 1254ms, sequenceid=563, compaction requested=true 2024-11-07T12:42:55,857 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ffbbe32019fd517ea21eab76e142aa7f: 2024-11-07T12:42:55,857 INFO [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:55,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffbbe32019fd517ea21eab76e142aa7f:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:42:55,857 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:55,857 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. because compaction request was cancelled 2024-11-07T12:42:55,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:42:55,857 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffbbe32019fd517ea21eab76e142aa7f:A 2024-11-07T12:42:55,857 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. after waiting 0 ms 2024-11-07T12:42:55,857 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:55,857 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. because compaction request was cancelled 2024-11-07T12:42:55,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffbbe32019fd517ea21eab76e142aa7f:B, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:42:55,857 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffbbe32019fd517ea21eab76e142aa7f:B 2024-11-07T12:42:55,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:42:55,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffbbe32019fd517ea21eab76e142aa7f:C, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:42:55,857 INFO [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(2902): Flushing ffbbe32019fd517ea21eab76e142aa7f 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-07T12:42:55,857 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. because compaction request was cancelled 2024-11-07T12:42:55,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:42:55,857 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffbbe32019fd517ea21eab76e142aa7f:C 2024-11-07T12:42:55,858 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffbbe32019fd517ea21eab76e142aa7f, store=A 2024-11-07T12:42:55,858 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:55,858 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffbbe32019fd517ea21eab76e142aa7f, store=B 2024-11-07T12:42:55,858 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:55,858 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffbbe32019fd517ea21eab76e142aa7f, store=C 2024-11-07T12:42:55,858 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:42:55,863 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411078b08c5e29cf640649d4d3653d118721c_ffbbe32019fd517ea21eab76e142aa7f is 50, key is test_row_0/A:col10/1730983375082/Put/seqid=0 2024-11-07T12:42:55,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742346_1522 (size=12454) 2024-11-07T12:42:56,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-07T12:42:56,273 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:56,277 INFO [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411078b08c5e29cf640649d4d3653d118721c_ffbbe32019fd517ea21eab76e142aa7f to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411078b08c5e29cf640649d4d3653d118721c_ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:56,277 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/2cde8ad2cccc40e8832ecfd2562ac065, store: [table=TestAcidGuarantees family=A region=ffbbe32019fd517ea21eab76e142aa7f] 2024-11-07T12:42:56,278 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/2cde8ad2cccc40e8832ecfd2562ac065 is 175, key is test_row_0/A:col10/1730983375082/Put/seqid=0 2024-11-07T12:42:56,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742347_1523 (size=31255) 2024-11-07T12:42:56,682 INFO [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=570, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/2cde8ad2cccc40e8832ecfd2562ac065 2024-11-07T12:42:56,687 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/59c3ed680226494ea1af76ec40e29c40 is 50, key is test_row_0/B:col10/1730983375082/Put/seqid=0 2024-11-07T12:42:56,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742348_1524 (size=12301) 2024-11-07T12:42:57,091 INFO [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=570 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/59c3ed680226494ea1af76ec40e29c40 2024-11-07T12:42:57,097 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/a26bf7be5f584a039aad4f7d86e09d21 is 50, key is test_row_0/C:col10/1730983375082/Put/seqid=0 2024-11-07T12:42:57,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742349_1525 (size=12301) 2024-11-07T12:42:57,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-07T12:42:57,501 INFO [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=570 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/a26bf7be5f584a039aad4f7d86e09d21 2024-11-07T12:42:57,505 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/A/2cde8ad2cccc40e8832ecfd2562ac065 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/2cde8ad2cccc40e8832ecfd2562ac065 2024-11-07T12:42:57,508 INFO [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/2cde8ad2cccc40e8832ecfd2562ac065, entries=150, sequenceid=570, filesize=30.5 K 2024-11-07T12:42:57,509 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/B/59c3ed680226494ea1af76ec40e29c40 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/59c3ed680226494ea1af76ec40e29c40 2024-11-07T12:42:57,512 INFO [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/59c3ed680226494ea1af76ec40e29c40, entries=150, sequenceid=570, filesize=12.0 K 2024-11-07T12:42:57,513 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/.tmp/C/a26bf7be5f584a039aad4f7d86e09d21 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/a26bf7be5f584a039aad4f7d86e09d21 2024-11-07T12:42:57,516 INFO [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/a26bf7be5f584a039aad4f7d86e09d21, entries=150, sequenceid=570, filesize=12.0 K 2024-11-07T12:42:57,517 INFO [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for ffbbe32019fd517ea21eab76e142aa7f in 1660ms, sequenceid=570, compaction requested=true 2024-11-07T12:42:57,517 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/dc3105dfbb8442f2aacd7173a87e4e25, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/de98c2784445452dabe5016d7eb58d9a, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/3500be1a50634c20ada4f06730f47d8e, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/54228504239146e5b1cd5b0d3709c1e6, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/0dfb744c68ad4fb98ed8363fb434ba1b, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/a84c8dae672c4f99a58382f672f3af10, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/26bc571ba9af40b783df4847b1de8007, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/4a4f40f77b934622ae165cdc19f6c6cf, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/e55dc0831f6b4de384dbccf65efd9e9e, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/560b7b82979f40dba212c1c24b3b3895, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/59243edfc3594cf79d701d7ce848061f, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/ccc7bc899cda46ada28346d351973b46, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/76c41d13c81f4961af8f6f0465b2dba1, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/fa9e7da4d0c14ebf92cfcb6b8936146f, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/ccabddd91a804b9f861640b5380bafed, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/39de740b13244d4ebd4ca237d90fec5a, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/97cf8893e9674643930515cd53522401, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/553fb7f407c74aff9cbc4445fc96f9e2, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/dd7185696b644f2eae55aa1beff625ff, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/360be99d55d84b7eafd6fdb6cd034dbf, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/c6d49ce7c5604aecb3f86cd10ed6e871, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/e27545bdfa8e4303b2b559f30a96ee87, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/240ca42cbde849edae557294bd9f06fc, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/8fc93f8cb7894943aff8e090cecd35a3, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/cc68a3e0dd274d498844ee92198dbd92, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/28a37e937a92412bb189a2b842aff785, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/d10040eafe1a4aea8c4ecd1e8000a782, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/39f22b17b5e842f98bba3553f90793d3, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/05524e9858b241f090d08b67efbba432, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/36e7b9626ff04db3a64c5689a23b06c7, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/5c56abc2f5c04f05925b72798ba90f69] to archive 2024-11-07T12:42:57,518 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T12:42:57,520 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/dc3105dfbb8442f2aacd7173a87e4e25 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/dc3105dfbb8442f2aacd7173a87e4e25 2024-11-07T12:42:57,521 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/de98c2784445452dabe5016d7eb58d9a to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/de98c2784445452dabe5016d7eb58d9a 2024-11-07T12:42:57,522 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/3500be1a50634c20ada4f06730f47d8e to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/3500be1a50634c20ada4f06730f47d8e 2024-11-07T12:42:57,523 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/54228504239146e5b1cd5b0d3709c1e6 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/54228504239146e5b1cd5b0d3709c1e6 2024-11-07T12:42:57,524 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/0dfb744c68ad4fb98ed8363fb434ba1b to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/0dfb744c68ad4fb98ed8363fb434ba1b 2024-11-07T12:42:57,525 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/a84c8dae672c4f99a58382f672f3af10 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/a84c8dae672c4f99a58382f672f3af10 2024-11-07T12:42:57,526 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/26bc571ba9af40b783df4847b1de8007 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/26bc571ba9af40b783df4847b1de8007 2024-11-07T12:42:57,527 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/4a4f40f77b934622ae165cdc19f6c6cf to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/4a4f40f77b934622ae165cdc19f6c6cf 2024-11-07T12:42:57,529 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/e55dc0831f6b4de384dbccf65efd9e9e to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/e55dc0831f6b4de384dbccf65efd9e9e 2024-11-07T12:42:57,532 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/560b7b82979f40dba212c1c24b3b3895 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/560b7b82979f40dba212c1c24b3b3895 2024-11-07T12:42:57,533 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/59243edfc3594cf79d701d7ce848061f to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/59243edfc3594cf79d701d7ce848061f 2024-11-07T12:42:57,534 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/ccc7bc899cda46ada28346d351973b46 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/ccc7bc899cda46ada28346d351973b46 2024-11-07T12:42:57,536 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/76c41d13c81f4961af8f6f0465b2dba1 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/76c41d13c81f4961af8f6f0465b2dba1 2024-11-07T12:42:57,537 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/fa9e7da4d0c14ebf92cfcb6b8936146f to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/fa9e7da4d0c14ebf92cfcb6b8936146f 2024-11-07T12:42:57,538 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/ccabddd91a804b9f861640b5380bafed to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/ccabddd91a804b9f861640b5380bafed 2024-11-07T12:42:57,539 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/39de740b13244d4ebd4ca237d90fec5a to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/39de740b13244d4ebd4ca237d90fec5a 2024-11-07T12:42:57,540 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/97cf8893e9674643930515cd53522401 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/97cf8893e9674643930515cd53522401 2024-11-07T12:42:57,541 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/553fb7f407c74aff9cbc4445fc96f9e2 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/553fb7f407c74aff9cbc4445fc96f9e2 2024-11-07T12:42:57,542 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/dd7185696b644f2eae55aa1beff625ff to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/dd7185696b644f2eae55aa1beff625ff 2024-11-07T12:42:57,543 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/360be99d55d84b7eafd6fdb6cd034dbf to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/360be99d55d84b7eafd6fdb6cd034dbf 2024-11-07T12:42:57,543 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/c6d49ce7c5604aecb3f86cd10ed6e871 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/c6d49ce7c5604aecb3f86cd10ed6e871 2024-11-07T12:42:57,544 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/e27545bdfa8e4303b2b559f30a96ee87 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/e27545bdfa8e4303b2b559f30a96ee87 2024-11-07T12:42:57,546 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/240ca42cbde849edae557294bd9f06fc to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/240ca42cbde849edae557294bd9f06fc 2024-11-07T12:42:57,547 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/8fc93f8cb7894943aff8e090cecd35a3 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/8fc93f8cb7894943aff8e090cecd35a3 2024-11-07T12:42:57,548 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/cc68a3e0dd274d498844ee92198dbd92 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/cc68a3e0dd274d498844ee92198dbd92 2024-11-07T12:42:57,550 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/28a37e937a92412bb189a2b842aff785 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/28a37e937a92412bb189a2b842aff785 2024-11-07T12:42:57,552 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/d10040eafe1a4aea8c4ecd1e8000a782 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/d10040eafe1a4aea8c4ecd1e8000a782 2024-11-07T12:42:57,556 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/39f22b17b5e842f98bba3553f90793d3 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/39f22b17b5e842f98bba3553f90793d3 2024-11-07T12:42:57,557 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/05524e9858b241f090d08b67efbba432 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/05524e9858b241f090d08b67efbba432 2024-11-07T12:42:57,558 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/36e7b9626ff04db3a64c5689a23b06c7 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/36e7b9626ff04db3a64c5689a23b06c7 2024-11-07T12:42:57,559 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/5c56abc2f5c04f05925b72798ba90f69 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/5c56abc2f5c04f05925b72798ba90f69 2024-11-07T12:42:57,561 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/de9b805f732a4297a0e8f31367965300, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/f2f002b36f204464820f32bccb430f65, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/ad161989282d40e282c6622f40da2d0b, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/82c0b841a3b04d40a87dc1dd14832e6c, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/bac4e360b16840bc9f3193ac02236cf3, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/44e4b025cfdc4ce299d491a6a515315f, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/c36e10399d0047bcbf6f1e69fe05ab5b, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/ac74714f4c604abaa8e2015d1ba05cd0, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/1dec3da7e84a4599a06d15462113ba7c, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/e8ed0332b6614cff9a58548047e5e52f, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/7c919ce66e0745dd9a3c683288bd86cf, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/4be8d8e8d0aa451b88004950b85e4070, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/540e32cb1b1a41b6a74340380f431840, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/a9e85e4fe3e44fc29aa8dd416e4418ed, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/4aa685ff56f24ff6ad79594995c1d75b, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/a3531242048d4e3fbfd9663ae2875e0c, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/c3691d6c0606412d8710ee14052e779e, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/f6cd9e5795954b1da4597f3d769e0468, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/994877357ab24ace88cf7263cccf090c, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/c3a89f91f97e431b86409c2e08f824a3, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/9b9b38f97ebd47fe9d7e258f5800eaf0, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/72e7104b418b4372adcf83e97eced795, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/a590c3c3e280452ba75352d5adcc0355, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/b3632077c45442cfa4a6b6a5cc1ab324, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/1a34d5b7ceaa4bf59bcf202ae7f1cab3, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/825328eefa2243fd8974c98aefee7946, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/919b4dd71dfc4e2283da486e448fc3b2, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/b47e1d2e450e4e3faf04734e9843b063, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/83625c98d9e049b38899a3d5ba6dd9ec, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/d2fed702c6e944abb64a713697d1b1e0, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/8b4c087d29bf4b4493695f02ac8a1bb6] to archive 2024-11-07T12:42:57,562 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T12:42:57,564 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/de9b805f732a4297a0e8f31367965300 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/de9b805f732a4297a0e8f31367965300 2024-11-07T12:42:57,565 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/f2f002b36f204464820f32bccb430f65 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/f2f002b36f204464820f32bccb430f65 2024-11-07T12:42:57,565 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/ad161989282d40e282c6622f40da2d0b to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/ad161989282d40e282c6622f40da2d0b 2024-11-07T12:42:57,566 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/82c0b841a3b04d40a87dc1dd14832e6c to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/82c0b841a3b04d40a87dc1dd14832e6c 2024-11-07T12:42:57,567 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/bac4e360b16840bc9f3193ac02236cf3 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/bac4e360b16840bc9f3193ac02236cf3 2024-11-07T12:42:57,568 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/44e4b025cfdc4ce299d491a6a515315f to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/44e4b025cfdc4ce299d491a6a515315f 2024-11-07T12:42:57,569 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/c36e10399d0047bcbf6f1e69fe05ab5b to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/c36e10399d0047bcbf6f1e69fe05ab5b 2024-11-07T12:42:57,570 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/ac74714f4c604abaa8e2015d1ba05cd0 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/ac74714f4c604abaa8e2015d1ba05cd0 2024-11-07T12:42:57,571 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/1dec3da7e84a4599a06d15462113ba7c to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/1dec3da7e84a4599a06d15462113ba7c 2024-11-07T12:42:57,572 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/e8ed0332b6614cff9a58548047e5e52f to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/e8ed0332b6614cff9a58548047e5e52f 2024-11-07T12:42:57,573 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/7c919ce66e0745dd9a3c683288bd86cf to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/7c919ce66e0745dd9a3c683288bd86cf 2024-11-07T12:42:57,574 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/4be8d8e8d0aa451b88004950b85e4070 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/4be8d8e8d0aa451b88004950b85e4070 2024-11-07T12:42:57,575 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/540e32cb1b1a41b6a74340380f431840 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/540e32cb1b1a41b6a74340380f431840 2024-11-07T12:42:57,576 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/a9e85e4fe3e44fc29aa8dd416e4418ed to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/a9e85e4fe3e44fc29aa8dd416e4418ed 2024-11-07T12:42:57,577 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/4aa685ff56f24ff6ad79594995c1d75b to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/4aa685ff56f24ff6ad79594995c1d75b 2024-11-07T12:42:57,578 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/a3531242048d4e3fbfd9663ae2875e0c to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/a3531242048d4e3fbfd9663ae2875e0c 2024-11-07T12:42:57,579 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/c3691d6c0606412d8710ee14052e779e to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/c3691d6c0606412d8710ee14052e779e 2024-11-07T12:42:57,581 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/f6cd9e5795954b1da4597f3d769e0468 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/f6cd9e5795954b1da4597f3d769e0468 2024-11-07T12:42:57,582 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/994877357ab24ace88cf7263cccf090c to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/994877357ab24ace88cf7263cccf090c 2024-11-07T12:42:57,584 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/c3a89f91f97e431b86409c2e08f824a3 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/c3a89f91f97e431b86409c2e08f824a3 2024-11-07T12:42:57,585 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/9b9b38f97ebd47fe9d7e258f5800eaf0 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/9b9b38f97ebd47fe9d7e258f5800eaf0 2024-11-07T12:42:57,586 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/72e7104b418b4372adcf83e97eced795 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/72e7104b418b4372adcf83e97eced795 2024-11-07T12:42:57,587 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/a590c3c3e280452ba75352d5adcc0355 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/a590c3c3e280452ba75352d5adcc0355 2024-11-07T12:42:57,588 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/b3632077c45442cfa4a6b6a5cc1ab324 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/b3632077c45442cfa4a6b6a5cc1ab324 2024-11-07T12:42:57,589 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/1a34d5b7ceaa4bf59bcf202ae7f1cab3 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/1a34d5b7ceaa4bf59bcf202ae7f1cab3 2024-11-07T12:42:57,590 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/825328eefa2243fd8974c98aefee7946 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/825328eefa2243fd8974c98aefee7946 2024-11-07T12:42:57,591 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/919b4dd71dfc4e2283da486e448fc3b2 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/919b4dd71dfc4e2283da486e448fc3b2 2024-11-07T12:42:57,592 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/b47e1d2e450e4e3faf04734e9843b063 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/b47e1d2e450e4e3faf04734e9843b063 2024-11-07T12:42:57,593 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/83625c98d9e049b38899a3d5ba6dd9ec to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/83625c98d9e049b38899a3d5ba6dd9ec 2024-11-07T12:42:57,594 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/d2fed702c6e944abb64a713697d1b1e0 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/d2fed702c6e944abb64a713697d1b1e0 2024-11-07T12:42:57,595 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/8b4c087d29bf4b4493695f02ac8a1bb6 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/8b4c087d29bf4b4493695f02ac8a1bb6 2024-11-07T12:42:57,597 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/4aaa81e78a504ad1be26b5dc7c1b4ba0, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/07ffca961d214d1f91d762d81dc0bec4, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/970c989c5c944fd3bb2a20b812f00291, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/c74ce60b8291448ba153fbfb20c6fba4, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/e782f5db518149b9a3d6e5f4e7efaca1, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/465ff595d6e34831b549dddba14c03e1, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/6504ced6f2b74f66a318c8d9bbe617b8, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/c2d1ad17ea484d5bb8c61d4da470e282, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/cef3cf9278b440ce83042b61928f3b61, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/faf2b5e3d27e48268ead25bea4d06f91, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/13f5e62eaefe43f1bd5c33ff327e0e09, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/1a81ab6b544a40c5b923e1e04549f763, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/7648d58a095140fa8ac33172dabcee71, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/16f8eacb8fdb4d31982ca6aa71a8b6ff, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/96ebafeecff1423888688a233f19b19a, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/8da116fb23314cdebd8461f50d83a03a, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/78e76d73414f4a7a981bdcf21bb49a07, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/1d106a1eacd74c869ae48dab987b4f81, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/efef0128b57f4b1690189720f4e5eb94, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/253b14d46edc47bcaf6ef0e89edb8f0d, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/577fe71a1f34454084f0fec48cbc1fec, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/88efdea78d8442b5a7aea59cc8b10a98, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/66365c60cf6f4df2b5c318e89a9eb16d, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/68623b16d2964570b8aefc12b98b0c95, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/9f0feb6f84d4477997d169b50d196cdd, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/65888d63555a46a69abe47c1e8c64ea0, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/3a268dbac01f4048b1a0a6bcba6f35c2, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/c6a872de2fd046a080cf8eb72d8cbb9c, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/eeba5c44a9c94ac6b5d91801471302a7, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/c56871bf91144dfabfb4be0508434c6f, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/611d6e99a7914503bb979433722e1659] to archive 2024-11-07T12:42:57,598 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T12:42:57,600 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/4aaa81e78a504ad1be26b5dc7c1b4ba0 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/4aaa81e78a504ad1be26b5dc7c1b4ba0 2024-11-07T12:42:57,601 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/07ffca961d214d1f91d762d81dc0bec4 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/07ffca961d214d1f91d762d81dc0bec4 2024-11-07T12:42:57,601 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/970c989c5c944fd3bb2a20b812f00291 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/970c989c5c944fd3bb2a20b812f00291 2024-11-07T12:42:57,602 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/c74ce60b8291448ba153fbfb20c6fba4 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/c74ce60b8291448ba153fbfb20c6fba4 2024-11-07T12:42:57,603 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/e782f5db518149b9a3d6e5f4e7efaca1 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/e782f5db518149b9a3d6e5f4e7efaca1 2024-11-07T12:42:57,604 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/465ff595d6e34831b549dddba14c03e1 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/465ff595d6e34831b549dddba14c03e1 2024-11-07T12:42:57,605 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/6504ced6f2b74f66a318c8d9bbe617b8 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/6504ced6f2b74f66a318c8d9bbe617b8 2024-11-07T12:42:57,606 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/c2d1ad17ea484d5bb8c61d4da470e282 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/c2d1ad17ea484d5bb8c61d4da470e282 2024-11-07T12:42:57,607 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/cef3cf9278b440ce83042b61928f3b61 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/cef3cf9278b440ce83042b61928f3b61 2024-11-07T12:42:57,608 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/faf2b5e3d27e48268ead25bea4d06f91 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/faf2b5e3d27e48268ead25bea4d06f91 2024-11-07T12:42:57,609 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/13f5e62eaefe43f1bd5c33ff327e0e09 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/13f5e62eaefe43f1bd5c33ff327e0e09 2024-11-07T12:42:57,610 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/1a81ab6b544a40c5b923e1e04549f763 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/1a81ab6b544a40c5b923e1e04549f763 2024-11-07T12:42:57,611 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/7648d58a095140fa8ac33172dabcee71 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/7648d58a095140fa8ac33172dabcee71 2024-11-07T12:42:57,612 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/16f8eacb8fdb4d31982ca6aa71a8b6ff to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/16f8eacb8fdb4d31982ca6aa71a8b6ff 2024-11-07T12:42:57,616 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/96ebafeecff1423888688a233f19b19a to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/96ebafeecff1423888688a233f19b19a 2024-11-07T12:42:57,617 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/8da116fb23314cdebd8461f50d83a03a to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/8da116fb23314cdebd8461f50d83a03a 2024-11-07T12:42:57,618 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/78e76d73414f4a7a981bdcf21bb49a07 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/78e76d73414f4a7a981bdcf21bb49a07 2024-11-07T12:42:57,619 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/1d106a1eacd74c869ae48dab987b4f81 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/1d106a1eacd74c869ae48dab987b4f81 2024-11-07T12:42:57,620 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/efef0128b57f4b1690189720f4e5eb94 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/efef0128b57f4b1690189720f4e5eb94 2024-11-07T12:42:57,621 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/253b14d46edc47bcaf6ef0e89edb8f0d to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/253b14d46edc47bcaf6ef0e89edb8f0d 2024-11-07T12:42:57,622 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/577fe71a1f34454084f0fec48cbc1fec to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/577fe71a1f34454084f0fec48cbc1fec 2024-11-07T12:42:57,623 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/88efdea78d8442b5a7aea59cc8b10a98 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/88efdea78d8442b5a7aea59cc8b10a98 2024-11-07T12:42:57,624 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/66365c60cf6f4df2b5c318e89a9eb16d to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/66365c60cf6f4df2b5c318e89a9eb16d 2024-11-07T12:42:57,625 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/68623b16d2964570b8aefc12b98b0c95 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/68623b16d2964570b8aefc12b98b0c95 2024-11-07T12:42:57,626 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/9f0feb6f84d4477997d169b50d196cdd to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/9f0feb6f84d4477997d169b50d196cdd 2024-11-07T12:42:57,626 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/65888d63555a46a69abe47c1e8c64ea0 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/65888d63555a46a69abe47c1e8c64ea0 2024-11-07T12:42:57,627 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/3a268dbac01f4048b1a0a6bcba6f35c2 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/3a268dbac01f4048b1a0a6bcba6f35c2 2024-11-07T12:42:57,628 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/c6a872de2fd046a080cf8eb72d8cbb9c to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/c6a872de2fd046a080cf8eb72d8cbb9c 2024-11-07T12:42:57,629 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/eeba5c44a9c94ac6b5d91801471302a7 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/eeba5c44a9c94ac6b5d91801471302a7 2024-11-07T12:42:57,630 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/c56871bf91144dfabfb4be0508434c6f to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/c56871bf91144dfabfb4be0508434c6f 2024-11-07T12:42:57,631 DEBUG [StoreCloser-TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/611d6e99a7914503bb979433722e1659 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/611d6e99a7914503bb979433722e1659 2024-11-07T12:42:57,635 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/recovered.edits/573.seqid, newMaxSeqId=573, maxSeqId=4 2024-11-07T12:42:57,636 INFO [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. 2024-11-07T12:42:57,636 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1676): Region close journal for ffbbe32019fd517ea21eab76e142aa7f: Waiting for close lock at 1730983375247Running coprocessor pre-close hooks at 1730983375247Disabling compacts and flushes for region at 1730983375247Disabling writes for close at 1730983375857 (+610 ms)Obtaining lock to block concurrent updates at 1730983375857Preparing flush snapshotting stores in ffbbe32019fd517ea21eab76e142aa7f at 1730983375857Finished memstore snapshotting TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1730983375858 (+1 ms)Flushing stores of TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f. at 1730983375858Flushing ffbbe32019fd517ea21eab76e142aa7f/A: creating writer at 1730983375858Flushing mob file ffbbe32019fd517ea21eab76e142aa7f/A: appending metadata at 1730983375863 (+5 ms)Flushing mob file ffbbe32019fd517ea21eab76e142aa7f/A: closing flushed file at 1730983375863Flushing ffbbe32019fd517ea21eab76e142aa7f/A: appending metadata at 1730983376277 (+414 ms)Flushing ffbbe32019fd517ea21eab76e142aa7f/A: closing flushed file at 1730983376277Flushing ffbbe32019fd517ea21eab76e142aa7f/B: creating writer at 1730983376685 (+408 ms)Flushing ffbbe32019fd517ea21eab76e142aa7f/B: appending metadata at 1730983376687 (+2 ms)Flushing ffbbe32019fd517ea21eab76e142aa7f/B: closing flushed file at 1730983376687Flushing ffbbe32019fd517ea21eab76e142aa7f/C: creating writer at 1730983377094 (+407 ms)Flushing ffbbe32019fd517ea21eab76e142aa7f/C: appending metadata at 1730983377097 (+3 ms)Flushing ffbbe32019fd517ea21eab76e142aa7f/C: closing flushed file at 1730983377097Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@50a6030e: reopening flushed file at 1730983377504 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@ce26f7c: reopening flushed file at 1730983377508 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6a222d8b: reopening flushed file at 1730983377512 (+4 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for ffbbe32019fd517ea21eab76e142aa7f in 1660ms, sequenceid=570, compaction requested=true at 1730983377517 (+5 ms)Writing region close event to WAL at 1730983377632 (+115 ms)Running coprocessor post-close hooks at 1730983377635 (+3 ms)Closed at 1730983377636 (+1 ms) 2024-11-07T12:42:57,638 INFO [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] handler.UnassignRegionHandler(157): Closed ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:57,638 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=121 updating hbase:meta row=ffbbe32019fd517ea21eab76e142aa7f, regionState=CLOSED 2024-11-07T12:42:57,639 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=122, ppid=121, state=RUNNABLE, hasLock=false; CloseRegionProcedure ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 because future has completed 2024-11-07T12:42:57,641 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-07T12:42:57,642 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; CloseRegionProcedure ffbbe32019fd517ea21eab76e142aa7f, server=db9ad1cb6cf9,34293,1730983260217 in 2.5450 sec 2024-11-07T12:42:57,643 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=121, resume processing ppid=120 2024-11-07T12:42:57,643 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, ppid=120, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ffbbe32019fd517ea21eab76e142aa7f, UNASSIGN in 2.5500 sec 2024-11-07T12:42:57,645 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-07T12:42:57,645 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.5540 sec 2024-11-07T12:42:57,646 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730983377646"}]},"ts":"1730983377646"} 2024-11-07T12:42:57,647 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-07T12:42:57,647 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-07T12:42:57,649 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.5620 sec 2024-11-07T12:42:57,837 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-07T12:42:59,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-07T12:42:59,262 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:42:59,263 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-07T12:42:59,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:42:59,265 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=123, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:42:59,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-07T12:42:59,266 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=123, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:42:59,268 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:59,271 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A, FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B, FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C, FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/recovered.edits] 2024-11-07T12:42:59,273 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/21cd23dc124045c49fa50fe07c4ea205 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/21cd23dc124045c49fa50fe07c4ea205 2024-11-07T12:42:59,274 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/2cde8ad2cccc40e8832ecfd2562ac065 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/2cde8ad2cccc40e8832ecfd2562ac065 2024-11-07T12:42:59,276 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/a1e7f09337d34167b98cd630679f73d3 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/a1e7f09337d34167b98cd630679f73d3 2024-11-07T12:42:59,277 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/b4ac82616b1d455fa4cdbc7025ebb8b5 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/A/b4ac82616b1d455fa4cdbc7025ebb8b5 2024-11-07T12:42:59,279 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/59c3ed680226494ea1af76ec40e29c40 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/59c3ed680226494ea1af76ec40e29c40 2024-11-07T12:42:59,281 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/a9f8c6189d844af1a84c0f14c1eebb2c to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/a9f8c6189d844af1a84c0f14c1eebb2c 2024-11-07T12:42:59,282 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/b1ecdda4a2f147c988c295e917688a4d to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/b1ecdda4a2f147c988c295e917688a4d 2024-11-07T12:42:59,283 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/dbaaec1a2ab24e539131e8d12003397e to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/B/dbaaec1a2ab24e539131e8d12003397e 2024-11-07T12:42:59,285 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/582efd5e85764639a97b2ac9328ce0c2 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/582efd5e85764639a97b2ac9328ce0c2 2024-11-07T12:42:59,286 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/a26bf7be5f584a039aad4f7d86e09d21 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/a26bf7be5f584a039aad4f7d86e09d21 2024-11-07T12:42:59,287 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/bf9207e8495b4ce69a76c48f8b56589a to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/bf9207e8495b4ce69a76c48f8b56589a 2024-11-07T12:42:59,288 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/d51c64bc099744148132fef5eaa2fb4c to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/C/d51c64bc099744148132fef5eaa2fb4c 2024-11-07T12:42:59,291 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/recovered.edits/573.seqid to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f/recovered.edits/573.seqid 2024-11-07T12:42:59,292 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:59,292 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-07T12:42:59,293 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-07T12:42:59,293 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-07T12:42:59,296 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107014348ffd8864f4eb21707cb8522f470_ffbbe32019fd517ea21eab76e142aa7f to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107014348ffd8864f4eb21707cb8522f470_ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:59,298 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110703969be376c94830bd5b7ccec3cbcef4_ffbbe32019fd517ea21eab76e142aa7f to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110703969be376c94830bd5b7ccec3cbcef4_ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:59,299 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107092d7e369cc748faad7a156de2f565dd_ffbbe32019fd517ea21eab76e142aa7f to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107092d7e369cc748faad7a156de2f565dd_ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:59,300 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411070d3bd09a54db427b8267cd4a1b93cd78_ffbbe32019fd517ea21eab76e142aa7f to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411070d3bd09a54db427b8267cd4a1b93cd78_ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:59,301 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411070eed8b74546643bd821237a9909f3ca1_ffbbe32019fd517ea21eab76e142aa7f to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411070eed8b74546643bd821237a9909f3ca1_ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:59,302 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411070f2868b06a7f4cacab30bbfb1d0df4c8_ffbbe32019fd517ea21eab76e142aa7f to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411070f2868b06a7f4cacab30bbfb1d0df4c8_ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:59,304 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411072bdab52aba86487799c64cbd22af82ee_ffbbe32019fd517ea21eab76e142aa7f to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411072bdab52aba86487799c64cbd22af82ee_ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:59,305 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110730340b52b2b8432d8ea79f0925a5ce86_ffbbe32019fd517ea21eab76e142aa7f to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110730340b52b2b8432d8ea79f0925a5ce86_ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:59,306 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107309c9464ab5b400c858d926330f156df_ffbbe32019fd517ea21eab76e142aa7f to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107309c9464ab5b400c858d926330f156df_ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:59,307 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110734c775a436134ce282a69d37dc365409_ffbbe32019fd517ea21eab76e142aa7f to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110734c775a436134ce282a69d37dc365409_ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:59,308 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411073f5aeceb55ab435b9ecbc632a5d38379_ffbbe32019fd517ea21eab76e142aa7f to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411073f5aeceb55ab435b9ecbc632a5d38379_ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:59,309 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411074d83c54d0f82455c955f02fefb659c1a_ffbbe32019fd517ea21eab76e142aa7f to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411074d83c54d0f82455c955f02fefb659c1a_ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:59,310 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110751b9948ca1ef47438149e92e17a0c2c9_ffbbe32019fd517ea21eab76e142aa7f to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110751b9948ca1ef47438149e92e17a0c2c9_ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:59,311 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110754796cc89aeb4d92920e080c75ea1e6a_ffbbe32019fd517ea21eab76e142aa7f to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110754796cc89aeb4d92920e080c75ea1e6a_ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:59,312 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110756e500440fa04922a5254279e2d4f61d_ffbbe32019fd517ea21eab76e142aa7f to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110756e500440fa04922a5254279e2d4f61d_ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:59,313 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110772b5580f9aef4b2b96136ef4ecb43435_ffbbe32019fd517ea21eab76e142aa7f to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110772b5580f9aef4b2b96136ef4ecb43435_ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:59,315 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110776cb73b25c054f50be2d9827a410abfe_ffbbe32019fd517ea21eab76e142aa7f to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110776cb73b25c054f50be2d9827a410abfe_ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:59,316 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411078a22a9ff67f94c2d80cf2e5e7112e6c4_ffbbe32019fd517ea21eab76e142aa7f to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411078a22a9ff67f94c2d80cf2e5e7112e6c4_ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:59,317 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411078b08c5e29cf640649d4d3653d118721c_ffbbe32019fd517ea21eab76e142aa7f to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411078b08c5e29cf640649d4d3653d118721c_ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:59,318 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411078c294f2cde414b478bfb3b8b3822497f_ffbbe32019fd517ea21eab76e142aa7f to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411078c294f2cde414b478bfb3b8b3822497f_ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:59,319 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107adec7d4aa1d04f3eac8667083e621085_ffbbe32019fd517ea21eab76e142aa7f to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107adec7d4aa1d04f3eac8667083e621085_ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:59,320 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107bb7cc13f77ed4875abf23d5ea3df69b8_ffbbe32019fd517ea21eab76e142aa7f to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107bb7cc13f77ed4875abf23d5ea3df69b8_ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:59,322 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107c4094e9929794487baa74ce1c000d337_ffbbe32019fd517ea21eab76e142aa7f to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107c4094e9929794487baa74ce1c000d337_ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:59,323 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107cc5bbd79ce4d48f9bf5f502f3790f67a_ffbbe32019fd517ea21eab76e142aa7f to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107cc5bbd79ce4d48f9bf5f502f3790f67a_ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:59,324 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107e89bba87e88a4faf9129c0f87ac1a5e7_ffbbe32019fd517ea21eab76e142aa7f to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107e89bba87e88a4faf9129c0f87ac1a5e7_ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:59,325 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107f456ad5095924509b6f4773a75a597fd_ffbbe32019fd517ea21eab76e142aa7f to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107f456ad5095924509b6f4773a75a597fd_ffbbe32019fd517ea21eab76e142aa7f 2024-11-07T12:42:59,326 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-07T12:42:59,329 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=123, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:42:59,332 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-07T12:42:59,334 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-07T12:42:59,335 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=123, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:42:59,335 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-07T12:42:59,336 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1730983379335"}]},"ts":"9223372036854775807"} 2024-11-07T12:42:59,338 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-07T12:42:59,338 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => ffbbe32019fd517ea21eab76e142aa7f, NAME => 'TestAcidGuarantees,,1730983350950.ffbbe32019fd517ea21eab76e142aa7f.', STARTKEY => '', ENDKEY => ''}] 2024-11-07T12:42:59,338 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-07T12:42:59,338 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1730983379338"}]},"ts":"9223372036854775807"} 2024-11-07T12:42:59,340 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-07T12:42:59,341 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=123, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:42:59,342 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 78 msec 2024-11-07T12:42:59,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-07T12:42:59,373 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-07T12:42:59,373 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:42:59,383 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=250 (was 252), OpenFileDescriptor=463 (was 465), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=985 (was 1073), ProcessCount=11 (was 11), AvailableMemoryMB=2925 (was 3080) 2024-11-07T12:42:59,394 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=250, OpenFileDescriptor=463, MaxFileDescriptor=1048576, SystemLoadAverage=985, ProcessCount=11, AvailableMemoryMB=2925 2024-11-07T12:42:59,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-07T12:42:59,396 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T12:42:59,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] procedure2.ProcedureExecutor(1139): Stored pid=124, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-07T12:42:59,398 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T12:42:59,398 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:42:59,398 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 124 2024-11-07T12:42:59,399 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T12:42:59,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-07T12:42:59,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742350_1526 (size=1061) 2024-11-07T12:42:59,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-07T12:42:59,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-07T12:42:59,718 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-07T12:42:59,809 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4 2024-11-07T12:42:59,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742351_1527 (size=53) 2024-11-07T12:43:00,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-07T12:43:00,223 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:43:00,223 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 850d939a0dd41b63931f314702af2ad5, disabling compactions & flushes 2024-11-07T12:43:00,223 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:00,224 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:00,224 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. after waiting 0 ms 2024-11-07T12:43:00,224 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:00,224 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:00,224 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 850d939a0dd41b63931f314702af2ad5: Waiting for close lock at 1730983380223Disabling compacts and flushes for region at 1730983380223Disabling writes for close at 1730983380224 (+1 ms)Writing region close event to WAL at 1730983380224Closed at 1730983380224 2024-11-07T12:43:00,225 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T12:43:00,225 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1730983380225"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730983380225"}]},"ts":"1730983380225"} 2024-11-07T12:43:00,227 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-07T12:43:00,228 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T12:43:00,228 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730983380228"}]},"ts":"1730983380228"} 2024-11-07T12:43:00,229 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-07T12:43:00,229 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=850d939a0dd41b63931f314702af2ad5, ASSIGN}] 2024-11-07T12:43:00,230 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=850d939a0dd41b63931f314702af2ad5, ASSIGN 2024-11-07T12:43:00,231 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=850d939a0dd41b63931f314702af2ad5, ASSIGN; state=OFFLINE, location=db9ad1cb6cf9,34293,1730983260217; forceNewPlan=false, retain=false 2024-11-07T12:43:00,382 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=125 updating hbase:meta row=850d939a0dd41b63931f314702af2ad5, regionState=OPENING, regionLocation=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:00,387 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=850d939a0dd41b63931f314702af2ad5, ASSIGN because future has completed 2024-11-07T12:43:00,388 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; OpenRegionProcedure 850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217}] 2024-11-07T12:43:00,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-07T12:43:00,544 INFO [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:00,544 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(7752): Opening region: {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} 2024-11-07T12:43:00,545 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 850d939a0dd41b63931f314702af2ad5 2024-11-07T12:43:00,545 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:43:00,545 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(7794): checking encryption for 850d939a0dd41b63931f314702af2ad5 2024-11-07T12:43:00,545 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(7797): checking classloading for 850d939a0dd41b63931f314702af2ad5 2024-11-07T12:43:00,548 INFO [StoreOpener-850d939a0dd41b63931f314702af2ad5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 850d939a0dd41b63931f314702af2ad5 2024-11-07T12:43:00,549 INFO [StoreOpener-850d939a0dd41b63931f314702af2ad5-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-07T12:43:00,550 INFO [StoreOpener-850d939a0dd41b63931f314702af2ad5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 850d939a0dd41b63931f314702af2ad5 columnFamilyName A 2024-11-07T12:43:00,550 DEBUG [StoreOpener-850d939a0dd41b63931f314702af2ad5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:00,550 INFO [StoreOpener-850d939a0dd41b63931f314702af2ad5-1 {}] regionserver.HStore(327): Store=850d939a0dd41b63931f314702af2ad5/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:43:00,550 INFO [StoreOpener-850d939a0dd41b63931f314702af2ad5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 850d939a0dd41b63931f314702af2ad5 2024-11-07T12:43:00,551 INFO [StoreOpener-850d939a0dd41b63931f314702af2ad5-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-07T12:43:00,552 INFO [StoreOpener-850d939a0dd41b63931f314702af2ad5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 850d939a0dd41b63931f314702af2ad5 columnFamilyName B 2024-11-07T12:43:00,552 DEBUG [StoreOpener-850d939a0dd41b63931f314702af2ad5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:00,552 INFO [StoreOpener-850d939a0dd41b63931f314702af2ad5-1 {}] regionserver.HStore(327): Store=850d939a0dd41b63931f314702af2ad5/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:43:00,552 INFO [StoreOpener-850d939a0dd41b63931f314702af2ad5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 850d939a0dd41b63931f314702af2ad5 2024-11-07T12:43:00,553 INFO [StoreOpener-850d939a0dd41b63931f314702af2ad5-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-07T12:43:00,553 INFO [StoreOpener-850d939a0dd41b63931f314702af2ad5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 850d939a0dd41b63931f314702af2ad5 columnFamilyName C 2024-11-07T12:43:00,553 DEBUG [StoreOpener-850d939a0dd41b63931f314702af2ad5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:00,554 INFO [StoreOpener-850d939a0dd41b63931f314702af2ad5-1 {}] regionserver.HStore(327): Store=850d939a0dd41b63931f314702af2ad5/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:43:00,554 INFO [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:00,554 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1038): replaying wal for 850d939a0dd41b63931f314702af2ad5 2024-11-07T12:43:00,555 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5 2024-11-07T12:43:00,555 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5 2024-11-07T12:43:00,556 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1048): stopping wal replay for 850d939a0dd41b63931f314702af2ad5 2024-11-07T12:43:00,556 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1060): Cleaning up temporary data for 850d939a0dd41b63931f314702af2ad5 2024-11-07T12:43:00,556 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-07T12:43:00,557 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1093): writing seq id for 850d939a0dd41b63931f314702af2ad5 2024-11-07T12:43:00,559 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T12:43:00,559 INFO [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1114): Opened 850d939a0dd41b63931f314702af2ad5; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70462409, jitterRate=0.04997171461582184}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-07T12:43:00,559 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 850d939a0dd41b63931f314702af2ad5 2024-11-07T12:43:00,560 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1006): Region open journal for 850d939a0dd41b63931f314702af2ad5: Running coprocessor pre-open hook at 1730983380545Writing region info on filesystem at 1730983380545Initializing all the Stores at 1730983380548 (+3 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730983380548Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730983380548Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730983380548Cleaning up temporary data from old regions at 1730983380556 (+8 ms)Running coprocessor post-open hooks at 1730983380559 (+3 ms)Region opened successfully at 1730983380560 (+1 ms) 2024-11-07T12:43:00,560 INFO [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., pid=126, masterSystemTime=1730983380540 2024-11-07T12:43:00,562 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:00,562 INFO [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:00,563 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=125 updating hbase:meta row=850d939a0dd41b63931f314702af2ad5, regionState=OPEN, openSeqNum=2, regionLocation=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:00,564 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=126, ppid=125, state=RUNNABLE, hasLock=false; OpenRegionProcedure 850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 because future has completed 2024-11-07T12:43:00,567 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-07T12:43:00,567 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; OpenRegionProcedure 850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 in 177 msec 2024-11-07T12:43:00,569 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=125, resume processing ppid=124 2024-11-07T12:43:00,569 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, ppid=124, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=850d939a0dd41b63931f314702af2ad5, ASSIGN in 338 msec 2024-11-07T12:43:00,570 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T12:43:00,570 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730983380570"}]},"ts":"1730983380570"} 2024-11-07T12:43:00,571 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-07T12:43:00,572 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T12:43:00,574 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1760 sec 2024-11-07T12:43:01,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-07T12:43:01,542 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:43:01,543 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d6d55bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:43:01,543 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request db9ad1cb6cf9,38619,-1 for getting cluster id 2024-11-07T12:43:01,543 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:43:01,544 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e542096-4b0e-4b01-9952-662501fc0f65' 2024-11-07T12:43:01,545 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:43:01,545 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e542096-4b0e-4b01-9952-662501fc0f65" 2024-11-07T12:43:01,545 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58e1cddc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:43:01,545 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [db9ad1cb6cf9,38619,-1] 2024-11-07T12:43:01,545 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:43:01,545 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:43:01,546 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46224, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:43:01,547 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46ea4578, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:43:01,547 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:43:01,548 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=-1] 2024-11-07T12:43:01,548 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:43:01,549 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41678, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:43:01,550 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-07T12:43:01,550 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is db9ad1cb6cf9,38619,1730983259119 2024-11-07T12:43:01,550 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@40084c9c 2024-11-07T12:43:01,551 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-07T12:43:01,551 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46240, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-07T12:43:01,553 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f0c05eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:43:01,553 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request db9ad1cb6cf9,38619,-1 for getting cluster id 2024-11-07T12:43:01,553 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:43:01,553 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e542096-4b0e-4b01-9952-662501fc0f65' 2024-11-07T12:43:01,553 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:43:01,553 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e542096-4b0e-4b01-9952-662501fc0f65" 2024-11-07T12:43:01,554 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c25cdaa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:43:01,554 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [db9ad1cb6cf9,38619,-1] 2024-11-07T12:43:01,554 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:43:01,554 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:43:01,555 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46258, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:43:01,555 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d173936, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:43:01,556 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79d7458, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:43:01,556 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request db9ad1cb6cf9,38619,-1 for getting cluster id 2024-11-07T12:43:01,556 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:43:01,556 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e542096-4b0e-4b01-9952-662501fc0f65' 2024-11-07T12:43:01,556 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:43:01,556 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e542096-4b0e-4b01-9952-662501fc0f65" 2024-11-07T12:43:01,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@143a8269, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:43:01,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [db9ad1cb6cf9,38619,-1] 2024-11-07T12:43:01,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:43:01,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:43:01,557 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46276, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:43:01,558 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b232bb6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:43:01,558 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32c61cf9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:43:01,558 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request db9ad1cb6cf9,38619,-1 for getting cluster id 2024-11-07T12:43:01,558 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:43:01,559 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e542096-4b0e-4b01-9952-662501fc0f65' 2024-11-07T12:43:01,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:43:01,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e542096-4b0e-4b01-9952-662501fc0f65" 2024-11-07T12:43:01,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24e28a66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:43:01,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [db9ad1cb6cf9,38619,-1] 2024-11-07T12:43:01,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:43:01,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:43:01,560 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46292, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:43:01,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bf06ff2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:43:01,561 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c296bf5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:43:01,561 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request db9ad1cb6cf9,38619,-1 for getting cluster id 2024-11-07T12:43:01,561 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:43:01,562 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e542096-4b0e-4b01-9952-662501fc0f65' 2024-11-07T12:43:01,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:43:01,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e542096-4b0e-4b01-9952-662501fc0f65" 2024-11-07T12:43:01,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ab7f240, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:43:01,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [db9ad1cb6cf9,38619,-1] 2024-11-07T12:43:01,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:43:01,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:43:01,563 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46302, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:43:01,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e69a3f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:43:01,564 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47bc1e91, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:43:01,564 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request db9ad1cb6cf9,38619,-1 for getting cluster id 2024-11-07T12:43:01,564 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:43:01,564 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e542096-4b0e-4b01-9952-662501fc0f65' 2024-11-07T12:43:01,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:43:01,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e542096-4b0e-4b01-9952-662501fc0f65" 2024-11-07T12:43:01,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6afd9bcc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:43:01,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [db9ad1cb6cf9,38619,-1] 2024-11-07T12:43:01,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:43:01,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:43:01,566 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46316, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:43:01,566 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5576268f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:43:01,567 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@672fe15e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:43:01,567 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request db9ad1cb6cf9,38619,-1 for getting cluster id 2024-11-07T12:43:01,567 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:43:01,567 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e542096-4b0e-4b01-9952-662501fc0f65' 2024-11-07T12:43:01,567 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:43:01,568 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e542096-4b0e-4b01-9952-662501fc0f65" 2024-11-07T12:43:01,568 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cee552, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:43:01,568 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [db9ad1cb6cf9,38619,-1] 2024-11-07T12:43:01,568 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:43:01,568 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:43:01,569 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46324, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:43:01,569 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26f2760f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:43:01,570 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71238b4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:43:01,570 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request db9ad1cb6cf9,38619,-1 for getting cluster id 2024-11-07T12:43:01,570 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:43:01,571 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e542096-4b0e-4b01-9952-662501fc0f65' 2024-11-07T12:43:01,571 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:43:01,571 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e542096-4b0e-4b01-9952-662501fc0f65" 2024-11-07T12:43:01,571 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cd1ac78, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:43:01,571 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [db9ad1cb6cf9,38619,-1] 2024-11-07T12:43:01,571 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:43:01,572 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:43:01,572 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46352, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:43:01,573 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b6933f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:43:01,573 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@625b319d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:43:01,573 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request db9ad1cb6cf9,38619,-1 for getting cluster id 2024-11-07T12:43:01,573 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:43:01,574 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e542096-4b0e-4b01-9952-662501fc0f65' 2024-11-07T12:43:01,574 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:43:01,574 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e542096-4b0e-4b01-9952-662501fc0f65" 2024-11-07T12:43:01,574 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d24efbd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:43:01,574 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [db9ad1cb6cf9,38619,-1] 2024-11-07T12:43:01,574 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:43:01,575 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:43:01,575 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46364, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:43:01,576 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28372096, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:43:01,576 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49cd7b43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:43:01,577 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request db9ad1cb6cf9,38619,-1 for getting cluster id 2024-11-07T12:43:01,577 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:43:01,577 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e542096-4b0e-4b01-9952-662501fc0f65' 2024-11-07T12:43:01,577 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:43:01,577 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e542096-4b0e-4b01-9952-662501fc0f65" 2024-11-07T12:43:01,578 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7282755f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:43:01,578 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [db9ad1cb6cf9,38619,-1] 2024-11-07T12:43:01,578 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:43:01,578 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:43:01,579 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46378, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:43:01,579 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f221da8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:43:01,585 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73f5639f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:43:01,585 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request db9ad1cb6cf9,38619,-1 for getting cluster id 2024-11-07T12:43:01,585 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:43:01,585 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e542096-4b0e-4b01-9952-662501fc0f65' 2024-11-07T12:43:01,585 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:43:01,586 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e542096-4b0e-4b01-9952-662501fc0f65" 2024-11-07T12:43:01,586 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29c5c2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:43:01,586 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [db9ad1cb6cf9,38619,-1] 2024-11-07T12:43:01,586 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:43:01,586 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:43:01,587 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46382, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:43:01,587 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@426ae3f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:43:01,588 DEBUG [Thread-2326 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:43:01,588 DEBUG [Thread-2341 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:43:01,589 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=-1] 2024-11-07T12:43:01,589 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:43:01,589 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=-1] 2024-11-07T12:43:01,589 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:43:01,589 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:43:01,589 DEBUG [Thread-2330 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:43:01,590 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41694, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:43:01,591 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41708, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:43:01,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-07T12:43:01,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2] 2024-11-07T12:43:01,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-07T12:43:01,594 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:43:01,594 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=-1] 2024-11-07T12:43:01,594 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:43:01,594 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2] 2024-11-07T12:43:01,595 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41722, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:43:01,595 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:43:01,595 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:43:01,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2] 2024-11-07T12:43:01,601 DEBUG [Thread-2328 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:43:01,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 850d939a0dd41b63931f314702af2ad5 2024-11-07T12:43:01,603 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=-1] 2024-11-07T12:43:01,603 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:43:01,604 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 850d939a0dd41b63931f314702af2ad5 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-07T12:43:01,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=A 2024-11-07T12:43:01,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:01,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=B 2024-11-07T12:43:01,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:01,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=C 2024-11-07T12:43:01,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:01,606 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41730, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:43:01,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2] 2024-11-07T12:43:01,609 DEBUG [Thread-2343 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:43:01,610 DEBUG [Thread-2343 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=-1] 2024-11-07T12:43:01,610 DEBUG [Thread-2343 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:43:01,612 DEBUG [Thread-2337 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:43:01,614 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41740, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:43:01,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2] 2024-11-07T12:43:01,617 DEBUG [Thread-2345 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:43:01,617 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=-1] 2024-11-07T12:43:01,617 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:43:01,618 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41746, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:43:01,619 DEBUG [Thread-2337 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=-1] 2024-11-07T12:43:01,619 DEBUG [Thread-2337 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:43:01,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2] 2024-11-07T12:43:01,620 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41750, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:43:01,624 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2] 2024-11-07T12:43:01,641 DEBUG [Thread-2332 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:43:01,650 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/ff7ffdbcbb0d48229070491b1c47d133 is 50, key is test_row_0/A:col10/1730983381601/Put/seqid=0 2024-11-07T12:43:01,651 DEBUG [Thread-2334 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:43:01,651 DEBUG [Thread-2332 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=-1] 2024-11-07T12:43:01,651 DEBUG [Thread-2332 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:43:01,652 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=-1] 2024-11-07T12:43:01,652 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:43:01,653 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41752, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:43:01,653 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41764, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:43:01,655 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2] 2024-11-07T12:43:01,657 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2] 2024-11-07T12:43:01,665 DEBUG [Thread-2339 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:43:01,666 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=-1] 2024-11-07T12:43:01,666 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:43:01,667 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1730983381634, max timestamp=1730983381663Num uniques 859; 2024-11-07T12:43:01,667 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1730983381634, max timestamp=1730983381663Num uniques 859; 2024-11-07T12:43:01,667 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1730983381634, max timestamp=1730983381663Num uniques 859; 2024-11-07T12:43:01,667 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41778, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:43:01,667 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-07T12:43:01,667 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-07T12:43:01,667 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-07T12:43:01,670 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2] 2024-11-07T12:43:01,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742352_1528 (size=9657) 2024-11-07T12:43:01,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-07T12:43:01,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:01,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41730 deadline: 1730983441702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:01,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:01,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:01,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:01,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:01,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41694 deadline: 1730983441716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:01,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:01,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:01,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:01,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:01,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41752 deadline: 1730983441717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:01,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:01,735 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:01,735 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:01,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1730983441721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:01,735 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:01,735 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:01,735 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:01,735 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:01,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:01,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41722 deadline: 1730983441726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:01,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:01,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:01,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:01,748 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-07T12:43:01,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:01,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. as already flushing 2024-11-07T12:43:01,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:01,748 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:01,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:01,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:01,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:01,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41730 deadline: 1730983441832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:01,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:01,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:01,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:01,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:01,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1730983441842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:01,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:01,853 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:01,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41752 deadline: 1730983441842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:01,853 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:01,853 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:01,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:01,853 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:01,853 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:01,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41694 deadline: 1730983441842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:01,854 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:01,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:01,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:01,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:01,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:01,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41722 deadline: 1730983441852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:01,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:01,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:01,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:01,902 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-07T12:43:01,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:01,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. as already flushing 2024-11-07T12:43:01,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:01,902 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:01,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:01,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:01,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-07T12:43:02,056 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-07T12:43:02,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:02,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. as already flushing 2024-11-07T12:43:02,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:02,056 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:02,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:02,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:02,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:02,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41730 deadline: 1730983442051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:02,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:02,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:02,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:02,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:02,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41752 deadline: 1730983442062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:02,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:02,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1730983442062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:02,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:02,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41694 deadline: 1730983442062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:02,074 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:02,074 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:02,075 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:02,075 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:02,075 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:02,075 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:02,075 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:02,075 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:02,075 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:02,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:02,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41722 deadline: 1730983442071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:02,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:02,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:02,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:02,092 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/ff7ffdbcbb0d48229070491b1c47d133 2024-11-07T12:43:02,135 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/d04d499c570f4a759b33df3da8044bdc is 50, key is test_row_0/B:col10/1730983381601/Put/seqid=0 2024-11-07T12:43:02,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742353_1529 (size=9657) 2024-11-07T12:43:02,209 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-07T12:43:02,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:02,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. as already flushing 2024-11-07T12:43:02,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:02,209 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:02,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:02,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:02,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-07T12:43:02,369 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-07T12:43:02,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:02,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. as already flushing 2024-11-07T12:43:02,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:02,369 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:02,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:02,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:02,372 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:02,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41730 deadline: 1730983442370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:02,373 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:02,373 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:02,373 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:02,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:02,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1730983442385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:02,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:02,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41694 deadline: 1730983442385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:02,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:02,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:02,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:02,392 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:02,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:02,393 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:02,393 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:02,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41752 deadline: 1730983442387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:02,395 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:02,395 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:02,395 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:02,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:02,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41722 deadline: 1730983442391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:02,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:02,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:02,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:02,529 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-07T12:43:02,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:02,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. as already flushing 2024-11-07T12:43:02,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:02,530 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:02,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:02,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:02,551 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/d04d499c570f4a759b33df3da8044bdc 2024-11-07T12:43:02,596 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/0924f27e2a4947c499be70119cb3e0b6 is 50, key is test_row_0/C:col10/1730983381601/Put/seqid=0 2024-11-07T12:43:02,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742354_1530 (size=9657) 2024-11-07T12:43:02,637 WARN [regionserver/db9ad1cb6cf9:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 0 2024-11-07T12:43:02,697 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-07T12:43:02,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:02,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. as already flushing 2024-11-07T12:43:02,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:02,701 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:02,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:02,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:02,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-07T12:43:02,871 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-07T12:43:02,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:02,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. as already flushing 2024-11-07T12:43:02,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:02,871 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:02,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:02,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:02,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:02,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41730 deadline: 1730983442884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:02,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:02,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:02,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:02,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:02,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1730983442904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:02,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:02,917 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:02,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41752 deadline: 1730983442904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:02,917 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:02,917 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:02,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:02,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41694 deadline: 1730983442908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:02,918 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:02,918 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:02,918 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:02,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:02,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:02,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:02,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:02,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41722 deadline: 1730983442922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:02,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:02,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:02,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:03,031 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-07T12:43:03,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:03,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. as already flushing 2024-11-07T12:43:03,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:03,031 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:03,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:03,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:03,034 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/0924f27e2a4947c499be70119cb3e0b6 2024-11-07T12:43:03,069 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/ff7ffdbcbb0d48229070491b1c47d133 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/ff7ffdbcbb0d48229070491b1c47d133 2024-11-07T12:43:03,076 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/ff7ffdbcbb0d48229070491b1c47d133, entries=100, sequenceid=11, filesize=9.4 K 2024-11-07T12:43:03,078 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/d04d499c570f4a759b33df3da8044bdc as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/d04d499c570f4a759b33df3da8044bdc 2024-11-07T12:43:03,085 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/d04d499c570f4a759b33df3da8044bdc, entries=100, sequenceid=11, filesize=9.4 K 2024-11-07T12:43:03,085 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/0924f27e2a4947c499be70119cb3e0b6 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/0924f27e2a4947c499be70119cb3e0b6 2024-11-07T12:43:03,092 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/0924f27e2a4947c499be70119cb3e0b6, entries=100, sequenceid=11, filesize=9.4 K 2024-11-07T12:43:03,093 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 850d939a0dd41b63931f314702af2ad5 in 1490ms, sequenceid=11, compaction requested=false 2024-11-07T12:43:03,093 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 850d939a0dd41b63931f314702af2ad5: 2024-11-07T12:43:03,191 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T12:43:03,197 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-07T12:43:03,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:03,197 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 850d939a0dd41b63931f314702af2ad5 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-07T12:43:03,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=A 2024-11-07T12:43:03,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:43:03,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=B 2024-11-07T12:43:03,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:43:03,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=C 2024-11-07T12:43:03,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:43:03,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/50a0da3c2cda4668a71a2125b9b88859 is 50, key is test_row_0/A:col10/1730983381714/Put/seqid=0 2024-11-07T12:43:03,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742355_1531 (size=12001) 2024-11-07T12:43:03,251 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/50a0da3c2cda4668a71a2125b9b88859 2024-11-07T12:43:03,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/0ff4c794cd93426fae98328b170e285a is 50, key is test_row_0/B:col10/1730983381714/Put/seqid=0 2024-11-07T12:43:03,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742356_1532 (size=12001) 2024-11-07T12:43:03,271 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/0ff4c794cd93426fae98328b170e285a 2024-11-07T12:43:03,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/a97d7f6e75fa4e829c0f5fdb479b0db2 is 50, key is test_row_0/C:col10/1730983381714/Put/seqid=0 2024-11-07T12:43:03,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742357_1533 (size=12001) 2024-11-07T12:43:03,721 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/a97d7f6e75fa4e829c0f5fdb479b0db2 2024-11-07T12:43:03,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-07T12:43:03,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/50a0da3c2cda4668a71a2125b9b88859 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/50a0da3c2cda4668a71a2125b9b88859 2024-11-07T12:43:03,779 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/50a0da3c2cda4668a71a2125b9b88859, entries=150, sequenceid=46, filesize=11.7 K 2024-11-07T12:43:03,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/0ff4c794cd93426fae98328b170e285a as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/0ff4c794cd93426fae98328b170e285a 2024-11-07T12:43:03,784 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/0ff4c794cd93426fae98328b170e285a, entries=150, sequenceid=46, filesize=11.7 K 2024-11-07T12:43:03,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/a97d7f6e75fa4e829c0f5fdb479b0db2 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/a97d7f6e75fa4e829c0f5fdb479b0db2 2024-11-07T12:43:03,788 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/a97d7f6e75fa4e829c0f5fdb479b0db2, entries=150, sequenceid=46, filesize=11.7 K 2024-11-07T12:43:03,788 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=0 B/0 for 850d939a0dd41b63931f314702af2ad5 in 591ms, sequenceid=46, compaction requested=false 2024-11-07T12:43:03,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 850d939a0dd41b63931f314702af2ad5: 2024-11-07T12:43:03,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:03,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-07T12:43:03,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-07T12:43:03,792 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-07T12:43:03,792 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1950 sec 2024-11-07T12:43:03,794 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 2.2040 sec 2024-11-07T12:43:03,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 850d939a0dd41b63931f314702af2ad5 2024-11-07T12:43:03,930 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 850d939a0dd41b63931f314702af2ad5 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-07T12:43:03,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=A 2024-11-07T12:43:03,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:03,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=B 2024-11-07T12:43:03,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:03,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=C 2024-11-07T12:43:03,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:03,936 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/b9852c71c7aa4b3eb458adb00f377b0a is 50, key is test_row_0/A:col10/1730983383913/Put/seqid=0 2024-11-07T12:43:03,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742358_1534 (size=16681) 2024-11-07T12:43:03,992 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1730983383962, max timestamp=1730983383989Num uniques 859; 2024-11-07T12:43:03,992 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-07T12:43:03,993 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1730983383962, max timestamp=1730983383989Num uniques 859; 2024-11-07T12:43:03,993 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-07T12:43:03,995 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1730983383962, max timestamp=1730983383989Num uniques 859; 2024-11-07T12:43:03,995 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-07T12:43:04,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:04,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:04,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41730 deadline: 1730983444033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:04,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41694 deadline: 1730983444031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:04,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:04,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:04,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:04,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:04,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41722 deadline: 1730983444042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:04,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:04,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:04,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:04,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:04,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:04,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:04,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:04,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41752 deadline: 1730983444053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:04,056 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:04,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:04,056 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:04,056 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:04,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1730983444054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:04,056 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:04,056 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:04,056 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:04,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:04,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1730983444162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:04,173 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:04,173 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:04,173 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:04,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:04,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41752 deadline: 1730983444162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:04,174 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:04,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:04,174 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:04,174 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:04,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41694 deadline: 1730983444162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:04,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:04,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41722 deadline: 1730983444163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:04,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:04,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:04,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:04,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:04,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41730 deadline: 1730983444163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:04,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:04,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:04,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:04,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:04,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:04,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:04,347 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/b9852c71c7aa4b3eb458adb00f377b0a 2024-11-07T12:43:04,357 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/7f21af6315b14191a36cb732e4556b18 is 50, key is test_row_0/B:col10/1730983383913/Put/seqid=0 2024-11-07T12:43:04,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742359_1535 (size=12001) 2024-11-07T12:43:04,368 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/7f21af6315b14191a36cb732e4556b18 2024-11-07T12:43:04,376 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/03ee9cce38c04955841ceb4ea0e6abec is 50, key is test_row_0/C:col10/1730983383913/Put/seqid=0 2024-11-07T12:43:04,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742360_1536 (size=12001) 2024-11-07T12:43:04,383 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/03ee9cce38c04955841ceb4ea0e6abec 2024-11-07T12:43:04,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:04,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41722 deadline: 1730983444382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:04,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:04,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:04,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41730 deadline: 1730983444383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:04,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:04,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:04,385 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:04,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41694 deadline: 1730983444383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:04,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:04,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:04,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:04,386 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:04,386 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:04,386 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:04,391 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/b9852c71c7aa4b3eb458adb00f377b0a as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/b9852c71c7aa4b3eb458adb00f377b0a 2024-11-07T12:43:04,394 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/b9852c71c7aa4b3eb458adb00f377b0a, entries=250, sequenceid=56, filesize=16.3 K 2024-11-07T12:43:04,395 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/7f21af6315b14191a36cb732e4556b18 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/7f21af6315b14191a36cb732e4556b18 2024-11-07T12:43:04,398 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/7f21af6315b14191a36cb732e4556b18, entries=150, sequenceid=56, filesize=11.7 K 2024-11-07T12:43:04,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:04,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41752 deadline: 1730983444387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:04,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:04,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1730983444387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:04,400 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:04,400 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:04,400 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:04,401 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:04,401 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:04,401 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:04,401 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/03ee9cce38c04955841ceb4ea0e6abec as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/03ee9cce38c04955841ceb4ea0e6abec 2024-11-07T12:43:04,405 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/03ee9cce38c04955841ceb4ea0e6abec, entries=150, sequenceid=56, filesize=11.7 K 2024-11-07T12:43:04,406 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=138.65 KB/141980 for 850d939a0dd41b63931f314702af2ad5 in 476ms, sequenceid=56, compaction requested=true 2024-11-07T12:43:04,406 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 850d939a0dd41b63931f314702af2ad5: 2024-11-07T12:43:04,406 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:43:04,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 850d939a0dd41b63931f314702af2ad5:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:43:04,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:04,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 850d939a0dd41b63931f314702af2ad5:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:43:04,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:04,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 850d939a0dd41b63931f314702af2ad5:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:43:04,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:43:04,407 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:43:04,407 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38339 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:43:04,407 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): 850d939a0dd41b63931f314702af2ad5/A is initiating minor compaction (all files) 2024-11-07T12:43:04,407 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 850d939a0dd41b63931f314702af2ad5/A in TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:04,407 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/ff7ffdbcbb0d48229070491b1c47d133, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/50a0da3c2cda4668a71a2125b9b88859, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/b9852c71c7aa4b3eb458adb00f377b0a] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp, totalSize=37.4 K 2024-11-07T12:43:04,407 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:43:04,407 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 850d939a0dd41b63931f314702af2ad5/B is initiating minor compaction (all files) 2024-11-07T12:43:04,408 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 850d939a0dd41b63931f314702af2ad5/B in TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:04,408 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/d04d499c570f4a759b33df3da8044bdc, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/0ff4c794cd93426fae98328b170e285a, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/7f21af6315b14191a36cb732e4556b18] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp, totalSize=32.9 K 2024-11-07T12:43:04,408 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff7ffdbcbb0d48229070491b1c47d133, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1730983381600 2024-11-07T12:43:04,408 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting d04d499c570f4a759b33df3da8044bdc, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1730983381600 2024-11-07T12:43:04,408 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 50a0da3c2cda4668a71a2125b9b88859, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1730983381712 2024-11-07T12:43:04,408 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ff4c794cd93426fae98328b170e285a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1730983381712 2024-11-07T12:43:04,409 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9852c71c7aa4b3eb458adb00f377b0a, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1730983383913 2024-11-07T12:43:04,409 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f21af6315b14191a36cb732e4556b18, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1730983383913 2024-11-07T12:43:04,422 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 850d939a0dd41b63931f314702af2ad5#A#compaction#447 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:43:04,422 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 850d939a0dd41b63931f314702af2ad5#B#compaction#448 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:43:04,422 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/76c84375bb6948a8bb0e91d1c15c2efa is 50, key is test_row_0/A:col10/1730983383913/Put/seqid=0 2024-11-07T12:43:04,423 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/d88e9bbfc4714ee7be67304353c12338 is 50, key is test_row_0/B:col10/1730983383913/Put/seqid=0 2024-11-07T12:43:04,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742361_1537 (size=12104) 2024-11-07T12:43:04,439 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/76c84375bb6948a8bb0e91d1c15c2efa as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/76c84375bb6948a8bb0e91d1c15c2efa 2024-11-07T12:43:04,445 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 850d939a0dd41b63931f314702af2ad5/A of 850d939a0dd41b63931f314702af2ad5 into 76c84375bb6948a8bb0e91d1c15c2efa(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:43:04,445 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 850d939a0dd41b63931f314702af2ad5: 2024-11-07T12:43:04,445 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., storeName=850d939a0dd41b63931f314702af2ad5/A, priority=13, startTime=1730983384406; duration=0sec 2024-11-07T12:43:04,445 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:43:04,445 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 850d939a0dd41b63931f314702af2ad5:A 2024-11-07T12:43:04,445 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:43:04,446 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:43:04,446 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): 850d939a0dd41b63931f314702af2ad5/C is initiating minor compaction (all files) 2024-11-07T12:43:04,446 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 850d939a0dd41b63931f314702af2ad5/C in TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:04,446 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/0924f27e2a4947c499be70119cb3e0b6, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/a97d7f6e75fa4e829c0f5fdb479b0db2, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/03ee9cce38c04955841ceb4ea0e6abec] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp, totalSize=32.9 K 2024-11-07T12:43:04,447 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0924f27e2a4947c499be70119cb3e0b6, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1730983381600 2024-11-07T12:43:04,448 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting a97d7f6e75fa4e829c0f5fdb479b0db2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1730983381712 2024-11-07T12:43:04,448 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03ee9cce38c04955841ceb4ea0e6abec, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1730983383913 2024-11-07T12:43:04,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742362_1538 (size=12104) 2024-11-07T12:43:04,464 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/d88e9bbfc4714ee7be67304353c12338 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/d88e9bbfc4714ee7be67304353c12338 2024-11-07T12:43:04,466 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 850d939a0dd41b63931f314702af2ad5#C#compaction#449 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:43:04,467 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/f24d168431834d9bab9d414bc2006ddc is 50, key is test_row_0/C:col10/1730983383913/Put/seqid=0 2024-11-07T12:43:04,475 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 850d939a0dd41b63931f314702af2ad5/B of 850d939a0dd41b63931f314702af2ad5 into d88e9bbfc4714ee7be67304353c12338(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:43:04,475 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 850d939a0dd41b63931f314702af2ad5: 2024-11-07T12:43:04,475 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., storeName=850d939a0dd41b63931f314702af2ad5/B, priority=13, startTime=1730983384407; duration=0sec 2024-11-07T12:43:04,475 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:04,475 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 850d939a0dd41b63931f314702af2ad5:B 2024-11-07T12:43:04,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742363_1539 (size=12104) 2024-11-07T12:43:04,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 850d939a0dd41b63931f314702af2ad5 2024-11-07T12:43:04,703 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 850d939a0dd41b63931f314702af2ad5 3/3 column families, dataSize=145.36 KB heapSize=410.38 KB 2024-11-07T12:43:04,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=A 2024-11-07T12:43:04,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:43:04,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=B 2024-11-07T12:43:04,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:43:04,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=C 2024-11-07T12:43:04,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:43:04,710 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/79bba2e570794c018ba2c753dd14fa06 is 50, key is test_row_0/A:col10/1730983384701/Put/seqid=0 2024-11-07T12:43:04,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742364_1540 (size=14341) 2024-11-07T12:43:04,718 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/79bba2e570794c018ba2c753dd14fa06 2024-11-07T12:43:04,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:04,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41752 deadline: 1730983444712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:04,723 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:04,723 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:04,723 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:04,729 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/cdf20744350b42c5a9884fc374a933f4 is 50, key is test_row_0/B:col10/1730983384701/Put/seqid=0 2024-11-07T12:43:04,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:04,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41730 deadline: 1730983444721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:04,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:04,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41694 deadline: 1730983444722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:04,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:04,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:04,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:04,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:04,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:04,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41722 deadline: 1730983444723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:04,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:04,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:04,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:04,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:04,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1730983444723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:04,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:04,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:04,731 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:04,731 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:04,731 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:04,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742365_1541 (size=12001) 2024-11-07T12:43:04,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:04,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41730 deadline: 1730983444842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:04,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:04,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:04,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41694 deadline: 1730983444842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:04,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:04,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:04,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:04,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:04,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:04,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:04,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41722 deadline: 1730983444842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:04,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:04,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:04,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:04,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:04,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1730983444842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:04,858 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:04,858 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:04,858 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:04,889 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/f24d168431834d9bab9d414bc2006ddc as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/f24d168431834d9bab9d414bc2006ddc 2024-11-07T12:43:04,894 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 850d939a0dd41b63931f314702af2ad5/C of 850d939a0dd41b63931f314702af2ad5 into f24d168431834d9bab9d414bc2006ddc(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:43:04,894 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 850d939a0dd41b63931f314702af2ad5: 2024-11-07T12:43:04,894 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., storeName=850d939a0dd41b63931f314702af2ad5/C, priority=13, startTime=1730983384407; duration=0sec 2024-11-07T12:43:04,894 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:04,894 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 850d939a0dd41b63931f314702af2ad5:C 2024-11-07T12:43:05,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:05,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41730 deadline: 1730983445052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:05,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:05,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:05,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:05,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41694 deadline: 1730983445052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:05,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:05,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:05,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:05,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:05,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:05,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41722 deadline: 1730983445061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:05,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:05,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1730983445061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:05,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:05,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:05,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:05,072 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:05,072 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:05,072 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:05,135 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/cdf20744350b42c5a9884fc374a933f4 2024-11-07T12:43:05,143 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/2c7b7c5118e24ff7b1d99b6cd0d2a0c5 is 50, key is test_row_0/C:col10/1730983384701/Put/seqid=0 2024-11-07T12:43:05,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742366_1542 (size=12001) 2024-11-07T12:43:05,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:05,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41752 deadline: 1730983445231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:05,242 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:05,242 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:05,242 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:05,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:05,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41730 deadline: 1730983445371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:05,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:05,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41694 deadline: 1730983445371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:05,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:05,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:05,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:05,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:05,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:05,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:05,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:05,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41722 deadline: 1730983445382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:05,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:05,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1730983445382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:05,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:05,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:05,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:05,391 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:05,391 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:05,391 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:05,563 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/2c7b7c5118e24ff7b1d99b6cd0d2a0c5 2024-11-07T12:43:05,567 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/79bba2e570794c018ba2c753dd14fa06 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/79bba2e570794c018ba2c753dd14fa06 2024-11-07T12:43:05,571 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/79bba2e570794c018ba2c753dd14fa06, entries=200, sequenceid=95, filesize=14.0 K 2024-11-07T12:43:05,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/cdf20744350b42c5a9884fc374a933f4 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/cdf20744350b42c5a9884fc374a933f4 2024-11-07T12:43:05,575 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/cdf20744350b42c5a9884fc374a933f4, entries=150, sequenceid=95, filesize=11.7 K 2024-11-07T12:43:05,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/2c7b7c5118e24ff7b1d99b6cd0d2a0c5 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/2c7b7c5118e24ff7b1d99b6cd0d2a0c5 2024-11-07T12:43:05,582 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/2c7b7c5118e24ff7b1d99b6cd0d2a0c5, entries=150, sequenceid=95, filesize=11.7 K 2024-11-07T12:43:05,586 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~145.36 KB/148850, heapSize ~410.33 KB/420176, currentSize=40.25 KB/41220 for 850d939a0dd41b63931f314702af2ad5 in 883ms, sequenceid=95, compaction requested=false 2024-11-07T12:43:05,586 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 850d939a0dd41b63931f314702af2ad5: 2024-11-07T12:43:05,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-07T12:43:05,762 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:43:05,763 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:43:05,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-07T12:43:05,766 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:43:05,767 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:43:05,768 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:43:05,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-07T12:43:05,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-07T12:43:05,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 850d939a0dd41b63931f314702af2ad5 2024-11-07T12:43:05,894 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 850d939a0dd41b63931f314702af2ad5 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-07T12:43:05,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=A 2024-11-07T12:43:05,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:05,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=B 2024-11-07T12:43:05,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:05,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=C 2024-11-07T12:43:05,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:05,905 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/0a46eb4565094f078d600d81f915762e is 50, key is test_row_0/A:col10/1730983384720/Put/seqid=0 2024-11-07T12:43:05,921 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-07T12:43:05,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:05,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. as already flushing 2024-11-07T12:43:05,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:05,921 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:05,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:05,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:05,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742367_1543 (size=14341) 2024-11-07T12:43:05,959 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1730983385938, max timestamp=1730983385955Num uniques 859; 2024-11-07T12:43:05,959 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-07T12:43:05,959 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1730983385938, max timestamp=1730983385955Num uniques 859; 2024-11-07T12:43:05,959 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-07T12:43:05,960 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1730983385938, max timestamp=1730983385955Num uniques 859; 2024-11-07T12:43:05,960 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-07T12:43:06,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:06,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41722 deadline: 1730983446015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:06,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:06,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1730983446016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:06,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:06,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41694 deadline: 1730983446016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:06,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:06,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:06,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:06,031 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:06,031 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:06,031 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:06,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:06,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:06,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:06,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:06,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41730 deadline: 1730983446021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:06,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:06,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:06,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:06,081 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-07T12:43:06,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:06,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. as already flushing 2024-11-07T12:43:06,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:06,081 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:06,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:06,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-07T12:43:06,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:06,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:06,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41694 deadline: 1730983446131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:06,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:06,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:06,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:06,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:06,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1730983446142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:06,160 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:06,160 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:06,160 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:06,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:06,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41722 deadline: 1730983446142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:06,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:06,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:06,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:06,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:06,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41730 deadline: 1730983446142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:06,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:06,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:06,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:06,245 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-07T12:43:06,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:06,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. as already flushing 2024-11-07T12:43:06,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:06,245 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:06,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:06,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:06,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:06,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41752 deadline: 1730983446251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:06,264 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to db9ad1cb6cf9,34293,1730983260217 for 'test_row_2' in 850d939a0dd41b63931f314702af2ad5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:43:06,265 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:06,265 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:06,265 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:06,340 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=106 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/0a46eb4565094f078d600d81f915762e 2024-11-07T12:43:06,358 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/a63b789119614bd9b6a3f06155e7a592 is 50, key is test_row_0/B:col10/1730983384720/Put/seqid=0 2024-11-07T12:43:06,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742368_1544 (size=12001) 2024-11-07T12:43:06,364 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=106 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/a63b789119614bd9b6a3f06155e7a592 2024-11-07T12:43:06,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:06,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41694 deadline: 1730983446352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:06,369 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:06,369 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:06,369 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:06,386 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/27ca2d6c113e436b91e8e333eb527743 is 50, key is test_row_0/C:col10/1730983384720/Put/seqid=0 2024-11-07T12:43:06,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-07T12:43:06,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:06,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41730 deadline: 1730983446372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:06,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:06,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1730983446372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:06,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:06,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41722 deadline: 1730983446372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:06,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:06,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:06,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:06,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:06,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:06,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:06,397 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:06,397 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:06,397 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:06,401 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-07T12:43:06,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:06,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. as already flushing 2024-11-07T12:43:06,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:06,401 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:06,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:06,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:06,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742369_1545 (size=12001) 2024-11-07T12:43:06,405 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=106 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/27ca2d6c113e436b91e8e333eb527743 2024-11-07T12:43:06,409 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/0a46eb4565094f078d600d81f915762e as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/0a46eb4565094f078d600d81f915762e 2024-11-07T12:43:06,412 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/0a46eb4565094f078d600d81f915762e, entries=200, sequenceid=106, filesize=14.0 K 2024-11-07T12:43:06,413 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/a63b789119614bd9b6a3f06155e7a592 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/a63b789119614bd9b6a3f06155e7a592 2024-11-07T12:43:06,417 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/a63b789119614bd9b6a3f06155e7a592, entries=150, sequenceid=106, filesize=11.7 K 2024-11-07T12:43:06,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/27ca2d6c113e436b91e8e333eb527743 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/27ca2d6c113e436b91e8e333eb527743 2024-11-07T12:43:06,421 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/27ca2d6c113e436b91e8e333eb527743, entries=150, sequenceid=106, filesize=11.7 K 2024-11-07T12:43:06,423 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 850d939a0dd41b63931f314702af2ad5 in 529ms, sequenceid=106, compaction requested=true 2024-11-07T12:43:06,425 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 850d939a0dd41b63931f314702af2ad5: 2024-11-07T12:43:06,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 850d939a0dd41b63931f314702af2ad5:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:43:06,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:06,426 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:43:06,426 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:43:06,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 850d939a0dd41b63931f314702af2ad5:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:43:06,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:06,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 850d939a0dd41b63931f314702af2ad5:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:43:06,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:43:06,436 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40786 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:43:06,437 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): 850d939a0dd41b63931f314702af2ad5/A is initiating minor compaction (all files) 2024-11-07T12:43:06,437 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:43:06,437 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 850d939a0dd41b63931f314702af2ad5/A in TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:06,437 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 850d939a0dd41b63931f314702af2ad5/B is initiating minor compaction (all files) 2024-11-07T12:43:06,437 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/76c84375bb6948a8bb0e91d1c15c2efa, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/79bba2e570794c018ba2c753dd14fa06, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/0a46eb4565094f078d600d81f915762e] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp, totalSize=39.8 K 2024-11-07T12:43:06,437 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 850d939a0dd41b63931f314702af2ad5/B in TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:06,437 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/d88e9bbfc4714ee7be67304353c12338, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/cdf20744350b42c5a9884fc374a933f4, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/a63b789119614bd9b6a3f06155e7a592] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp, totalSize=35.3 K 2024-11-07T12:43:06,437 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76c84375bb6948a8bb0e91d1c15c2efa, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1730983383913 2024-11-07T12:43:06,438 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting d88e9bbfc4714ee7be67304353c12338, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1730983383913 2024-11-07T12:43:06,438 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79bba2e570794c018ba2c753dd14fa06, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1730983384038 2024-11-07T12:43:06,438 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting cdf20744350b42c5a9884fc374a933f4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1730983384051 2024-11-07T12:43:06,438 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a46eb4565094f078d600d81f915762e, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1730983384717 2024-11-07T12:43:06,439 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting a63b789119614bd9b6a3f06155e7a592, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1730983384719 2024-11-07T12:43:06,451 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 850d939a0dd41b63931f314702af2ad5#B#compaction#456 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:43:06,451 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 850d939a0dd41b63931f314702af2ad5#A#compaction#457 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:43:06,451 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/1d6082b99e304fc6b1494a4093179c65 is 50, key is test_row_0/A:col10/1730983384720/Put/seqid=0 2024-11-07T12:43:06,451 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/7541e610d3c1416fbac659eee5019d36 is 50, key is test_row_0/B:col10/1730983384720/Put/seqid=0 2024-11-07T12:43:06,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742371_1547 (size=12207) 2024-11-07T12:43:06,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742370_1546 (size=12207) 2024-11-07T12:43:06,562 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-07T12:43:06,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:06,563 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 850d939a0dd41b63931f314702af2ad5 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-07T12:43:06,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=A 2024-11-07T12:43:06,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:43:06,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=B 2024-11-07T12:43:06,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:43:06,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=C 2024-11-07T12:43:06,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:43:06,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/7507af023a764d05936b3228797013bf is 50, key is test_row_0/A:col10/1730983385995/Put/seqid=0 2024-11-07T12:43:06,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742372_1548 (size=12151) 2024-11-07T12:43:06,653 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/7507af023a764d05936b3228797013bf 2024-11-07T12:43:06,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/a5b46aaf1bf2488a822ac5b1b1f65154 is 50, key is test_row_0/B:col10/1730983385995/Put/seqid=0 2024-11-07T12:43:06,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 850d939a0dd41b63931f314702af2ad5 2024-11-07T12:43:06,681 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. as already flushing 2024-11-07T12:43:06,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742373_1549 (size=12151) 2024-11-07T12:43:06,715 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/a5b46aaf1bf2488a822ac5b1b1f65154 2024-11-07T12:43:06,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:06,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41722 deadline: 1730983446717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:06,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:06,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:06,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:06,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/5a29b9204dc94f8aa6e94116512cd122 is 50, key is test_row_0/C:col10/1730983385995/Put/seqid=0 2024-11-07T12:43:06,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:06,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1730983446727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:06,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:06,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41694 deadline: 1730983446728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:06,748 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:06,748 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:06,748 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:06,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:06,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:06,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:06,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:06,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41730 deadline: 1730983446737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:06,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:06,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:06,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:06,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742374_1550 (size=12151) 2024-11-07T12:43:06,770 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/5a29b9204dc94f8aa6e94116512cd122 2024-11-07T12:43:06,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/7507af023a764d05936b3228797013bf as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/7507af023a764d05936b3228797013bf 2024-11-07T12:43:06,786 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/7507af023a764d05936b3228797013bf, entries=150, sequenceid=142, filesize=11.9 K 2024-11-07T12:43:06,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/a5b46aaf1bf2488a822ac5b1b1f65154 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/a5b46aaf1bf2488a822ac5b1b1f65154 2024-11-07T12:43:06,793 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/a5b46aaf1bf2488a822ac5b1b1f65154, entries=150, sequenceid=142, filesize=11.9 K 2024-11-07T12:43:06,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/5a29b9204dc94f8aa6e94116512cd122 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/5a29b9204dc94f8aa6e94116512cd122 2024-11-07T12:43:06,798 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/5a29b9204dc94f8aa6e94116512cd122, entries=150, sequenceid=142, filesize=11.9 K 2024-11-07T12:43:06,799 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 850d939a0dd41b63931f314702af2ad5 in 237ms, sequenceid=142, compaction requested=true 2024-11-07T12:43:06,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 850d939a0dd41b63931f314702af2ad5: 2024-11-07T12:43:06,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:06,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-07T12:43:06,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-07T12:43:06,803 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-07T12:43:06,804 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0340 sec 2024-11-07T12:43:06,806 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.0420 sec 2024-11-07T12:43:06,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 850d939a0dd41b63931f314702af2ad5 2024-11-07T12:43:06,844 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 850d939a0dd41b63931f314702af2ad5 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-07T12:43:06,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=A 2024-11-07T12:43:06,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:06,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=B 2024-11-07T12:43:06,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:06,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=C 2024-11-07T12:43:06,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:06,851 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/13d09177d3774a3db0af3b089bb8bf15 is 50, key is test_row_0/A:col10/1730983386702/Put/seqid=0 2024-11-07T12:43:06,878 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/7541e610d3c1416fbac659eee5019d36 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/7541e610d3c1416fbac659eee5019d36 2024-11-07T12:43:06,883 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 850d939a0dd41b63931f314702af2ad5/B of 850d939a0dd41b63931f314702af2ad5 into 7541e610d3c1416fbac659eee5019d36(size=11.9 K), total size for store is 23.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:43:06,884 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 850d939a0dd41b63931f314702af2ad5: 2024-11-07T12:43:06,884 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., storeName=850d939a0dd41b63931f314702af2ad5/B, priority=13, startTime=1730983386426; duration=0sec 2024-11-07T12:43:06,884 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:43:06,884 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 850d939a0dd41b63931f314702af2ad5:B 2024-11-07T12:43:06,884 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:43:06,886 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:43:06,886 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 850d939a0dd41b63931f314702af2ad5/C is initiating minor compaction (all files) 2024-11-07T12:43:06,887 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 850d939a0dd41b63931f314702af2ad5/C in TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:06,887 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/f24d168431834d9bab9d414bc2006ddc, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/2c7b7c5118e24ff7b1d99b6cd0d2a0c5, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/27ca2d6c113e436b91e8e333eb527743, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/5a29b9204dc94f8aa6e94116512cd122] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp, totalSize=47.1 K 2024-11-07T12:43:06,888 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting f24d168431834d9bab9d414bc2006ddc, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1730983383913 2024-11-07T12:43:06,888 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/1d6082b99e304fc6b1494a4093179c65 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/1d6082b99e304fc6b1494a4093179c65 2024-11-07T12:43:06,888 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c7b7c5118e24ff7b1d99b6cd0d2a0c5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1730983384051 2024-11-07T12:43:06,889 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 27ca2d6c113e436b91e8e333eb527743, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1730983384719 2024-11-07T12:43:06,889 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a29b9204dc94f8aa6e94116512cd122, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1730983385987 2024-11-07T12:43:06,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742375_1551 (size=14541) 2024-11-07T12:43:06,895 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 850d939a0dd41b63931f314702af2ad5/A of 850d939a0dd41b63931f314702af2ad5 into 1d6082b99e304fc6b1494a4093179c65(size=11.9 K), total size for store is 23.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:43:06,895 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 850d939a0dd41b63931f314702af2ad5: 2024-11-07T12:43:06,895 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., storeName=850d939a0dd41b63931f314702af2ad5/A, priority=13, startTime=1730983386425; duration=0sec 2024-11-07T12:43:06,895 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:06,895 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 850d939a0dd41b63931f314702af2ad5:A 2024-11-07T12:43:06,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-07T12:43:06,902 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:43:06,903 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:43:06,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-07T12:43:06,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-07T12:43:06,909 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 850d939a0dd41b63931f314702af2ad5#C#compaction#462 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:43:06,910 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/ef5e053528fc4de794b3f37b3c697aa2 is 50, key is test_row_0/C:col10/1730983385995/Put/seqid=0 2024-11-07T12:43:06,911 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:43:06,915 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:43:06,915 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:43:06,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742376_1552 (size=12391) 2024-11-07T12:43:06,955 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1730983386915, max timestamp=1730983386951Num uniques 859; 2024-11-07T12:43:06,955 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-07T12:43:06,958 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1730983386915, max timestamp=1730983386951Num uniques 859; 2024-11-07T12:43:06,958 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-07T12:43:06,963 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1730983386929, max timestamp=1730983386951Num uniques 859; 2024-11-07T12:43:06,963 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-07T12:43:07,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-07T12:43:07,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:07,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41722 deadline: 1730983447020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:07,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:07,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41730 deadline: 1730983447020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:07,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:07,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:07,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41694 deadline: 1730983447022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:07,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:07,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:07,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:07,042 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:07,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1730983447023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:07,042 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:07,042 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:07,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:07,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:07,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:07,044 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:07,044 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:07,044 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:07,071 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-07T12:43:07,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:07,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. as already flushing 2024-11-07T12:43:07,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:07,071 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:07,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:07,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:07,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:07,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41730 deadline: 1730983447151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:07,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:07,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41694 deadline: 1730983447152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:07,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:07,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41722 deadline: 1730983447152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:07,156 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:07,156 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:07,156 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:07,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:07,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:07,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:07,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:07,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1730983447152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:07,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:07,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:07,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:07,156 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:07,156 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:07,156 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:07,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-07T12:43:07,227 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-07T12:43:07,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:07,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. as already flushing 2024-11-07T12:43:07,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:07,228 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:07,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:07,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:07,292 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/13d09177d3774a3db0af3b089bb8bf15 2024-11-07T12:43:07,302 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/48010982943b42e4bf3299d7d67a1a2a is 50, key is test_row_0/B:col10/1730983386702/Put/seqid=0 2024-11-07T12:43:07,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742377_1553 (size=12151) 2024-11-07T12:43:07,314 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/48010982943b42e4bf3299d7d67a1a2a 2024-11-07T12:43:07,321 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/5df70c77ac1e499c87de84f6dd8652e4 is 50, key is test_row_0/C:col10/1730983386702/Put/seqid=0 2024-11-07T12:43:07,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742378_1554 (size=12151) 2024-11-07T12:43:07,324 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/5df70c77ac1e499c87de84f6dd8652e4 2024-11-07T12:43:07,329 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/13d09177d3774a3db0af3b089bb8bf15 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/13d09177d3774a3db0af3b089bb8bf15 2024-11-07T12:43:07,333 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/13d09177d3774a3db0af3b089bb8bf15, entries=200, sequenceid=153, filesize=14.2 K 2024-11-07T12:43:07,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/48010982943b42e4bf3299d7d67a1a2a as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/48010982943b42e4bf3299d7d67a1a2a 2024-11-07T12:43:07,336 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/48010982943b42e4bf3299d7d67a1a2a, entries=150, sequenceid=153, filesize=11.9 K 2024-11-07T12:43:07,338 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/5df70c77ac1e499c87de84f6dd8652e4 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/5df70c77ac1e499c87de84f6dd8652e4 2024-11-07T12:43:07,343 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/5df70c77ac1e499c87de84f6dd8652e4, entries=150, sequenceid=153, filesize=11.9 K 2024-11-07T12:43:07,344 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 850d939a0dd41b63931f314702af2ad5 in 500ms, sequenceid=153, compaction requested=true 2024-11-07T12:43:07,344 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 850d939a0dd41b63931f314702af2ad5: 2024-11-07T12:43:07,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 850d939a0dd41b63931f314702af2ad5:A, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:43:07,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:07,344 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:43:07,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 850d939a0dd41b63931f314702af2ad5:B, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:43:07,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:43:07,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 850d939a0dd41b63931f314702af2ad5:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:43:07,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-07T12:43:07,345 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38899 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:43:07,345 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): 850d939a0dd41b63931f314702af2ad5/A is initiating minor compaction (all files) 2024-11-07T12:43:07,345 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 850d939a0dd41b63931f314702af2ad5/A in TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:07,345 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/1d6082b99e304fc6b1494a4093179c65, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/7507af023a764d05936b3228797013bf, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/13d09177d3774a3db0af3b089bb8bf15] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp, totalSize=38.0 K 2024-11-07T12:43:07,345 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d6082b99e304fc6b1494a4093179c65, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1730983384719 2024-11-07T12:43:07,345 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7507af023a764d05936b3228797013bf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1730983385987 2024-11-07T12:43:07,346 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13d09177d3774a3db0af3b089bb8bf15, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1730983386702 2024-11-07T12:43:07,353 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 850d939a0dd41b63931f314702af2ad5#A#compaction#465 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:43:07,354 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/c1e4851b3abb42159c2fb0eb7e7729b7 is 50, key is test_row_0/A:col10/1730983386702/Put/seqid=0 2024-11-07T12:43:07,360 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/ef5e053528fc4de794b3f37b3c697aa2 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/ef5e053528fc4de794b3f37b3c697aa2 2024-11-07T12:43:07,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742379_1555 (size=12459) 2024-11-07T12:43:07,370 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/c1e4851b3abb42159c2fb0eb7e7729b7 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/c1e4851b3abb42159c2fb0eb7e7729b7 2024-11-07T12:43:07,370 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 850d939a0dd41b63931f314702af2ad5/C of 850d939a0dd41b63931f314702af2ad5 into ef5e053528fc4de794b3f37b3c697aa2(size=12.1 K), total size for store is 24.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:43:07,370 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 850d939a0dd41b63931f314702af2ad5: 2024-11-07T12:43:07,370 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., storeName=850d939a0dd41b63931f314702af2ad5/C, priority=12, startTime=1730983386426; duration=0sec 2024-11-07T12:43:07,370 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-07T12:43:07,370 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 850d939a0dd41b63931f314702af2ad5:C 2024-11-07T12:43:07,370 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 850d939a0dd41b63931f314702af2ad5:C 2024-11-07T12:43:07,370 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:43:07,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 850d939a0dd41b63931f314702af2ad5 2024-11-07T12:43:07,371 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 850d939a0dd41b63931f314702af2ad5 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-07T12:43:07,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=A 2024-11-07T12:43:07,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:43:07,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=B 2024-11-07T12:43:07,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:43:07,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=C 2024-11-07T12:43:07,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:43:07,372 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:43:07,372 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 850d939a0dd41b63931f314702af2ad5/B is initiating minor compaction (all files) 2024-11-07T12:43:07,372 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 850d939a0dd41b63931f314702af2ad5/B in TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:07,372 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/7541e610d3c1416fbac659eee5019d36, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/a5b46aaf1bf2488a822ac5b1b1f65154, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/48010982943b42e4bf3299d7d67a1a2a] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp, totalSize=35.7 K 2024-11-07T12:43:07,373 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 7541e610d3c1416fbac659eee5019d36, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1730983384719 2024-11-07T12:43:07,373 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting a5b46aaf1bf2488a822ac5b1b1f65154, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1730983385987 2024-11-07T12:43:07,374 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 48010982943b42e4bf3299d7d67a1a2a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1730983386702 2024-11-07T12:43:07,376 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 850d939a0dd41b63931f314702af2ad5/A of 850d939a0dd41b63931f314702af2ad5 into c1e4851b3abb42159c2fb0eb7e7729b7(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:43:07,376 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 850d939a0dd41b63931f314702af2ad5: 2024-11-07T12:43:07,376 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., storeName=850d939a0dd41b63931f314702af2ad5/A, priority=13, startTime=1730983387344; duration=0sec 2024-11-07T12:43:07,376 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:43:07,376 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 850d939a0dd41b63931f314702af2ad5:A 2024-11-07T12:43:07,377 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-07T12:43:07,379 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/721f5882475c452ba689e2dd4a4736ef is 50, key is test_row_0/A:col10/1730983387019/Put/seqid=0 2024-11-07T12:43:07,380 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-07T12:43:07,380 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-07T12:43:07,381 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. because compaction request was cancelled 2024-11-07T12:43:07,381 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 850d939a0dd41b63931f314702af2ad5:C 2024-11-07T12:43:07,382 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-07T12:43:07,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:07,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. as already flushing 2024-11-07T12:43:07,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:07,382 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:07,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:07,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:07,391 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 850d939a0dd41b63931f314702af2ad5#B#compaction#467 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:43:07,391 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/96088dd6609140428ac6f53fca1852c9 is 50, key is test_row_0/B:col10/1730983386702/Put/seqid=0 2024-11-07T12:43:07,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742380_1556 (size=14541) 2024-11-07T12:43:07,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742381_1557 (size=12459) 2024-11-07T12:43:07,408 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/721f5882475c452ba689e2dd4a4736ef 2024-11-07T12:43:07,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:07,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41694 deadline: 1730983447398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:07,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:07,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:07,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:07,419 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/dad736474d52401298f14efec800557b is 50, key is test_row_0/B:col10/1730983387019/Put/seqid=0 2024-11-07T12:43:07,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:07,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41730 deadline: 1730983447398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:07,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:07,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:07,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:07,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:07,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41722 deadline: 1730983447398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:07,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:07,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:07,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:07,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:07,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1730983447406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:07,426 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:07,427 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:07,427 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:07,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742382_1558 (size=12151) 2024-11-07T12:43:07,432 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/dad736474d52401298f14efec800557b 2024-11-07T12:43:07,442 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/d722000c169245bc987716f59b6c21b5 is 50, key is test_row_0/C:col10/1730983387019/Put/seqid=0 2024-11-07T12:43:07,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742383_1559 (size=12151) 2024-11-07T12:43:07,456 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/d722000c169245bc987716f59b6c21b5 2024-11-07T12:43:07,460 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/721f5882475c452ba689e2dd4a4736ef as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/721f5882475c452ba689e2dd4a4736ef 2024-11-07T12:43:07,469 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/721f5882475c452ba689e2dd4a4736ef, entries=200, sequenceid=191, filesize=14.2 K 2024-11-07T12:43:07,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/dad736474d52401298f14efec800557b as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/dad736474d52401298f14efec800557b 2024-11-07T12:43:07,474 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/dad736474d52401298f14efec800557b, entries=150, sequenceid=191, filesize=11.9 K 2024-11-07T12:43:07,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/d722000c169245bc987716f59b6c21b5 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/d722000c169245bc987716f59b6c21b5 2024-11-07T12:43:07,479 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/d722000c169245bc987716f59b6c21b5, entries=150, sequenceid=191, filesize=11.9 K 2024-11-07T12:43:07,482 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for 850d939a0dd41b63931f314702af2ad5 in 111ms, sequenceid=191, compaction requested=true 2024-11-07T12:43:07,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 850d939a0dd41b63931f314702af2ad5: 2024-11-07T12:43:07,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 850d939a0dd41b63931f314702af2ad5:A, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:43:07,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:07,483 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-07T12:43:07,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 850d939a0dd41b63931f314702af2ad5:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:43:07,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:43:07,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 850d939a0dd41b63931f314702af2ad5:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:43:07,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-07T12:43:07,483 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-07T12:43:07,483 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-07T12:43:07,483 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. because compaction request was cancelled 2024-11-07T12:43:07,483 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 850d939a0dd41b63931f314702af2ad5:A 2024-11-07T12:43:07,483 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 3 compacting, 1 eligible, 16 blocking 2024-11-07T12:43:07,483 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-07T12:43:07,484 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-07T12:43:07,484 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. because compaction request was cancelled 2024-11-07T12:43:07,484 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 850d939a0dd41b63931f314702af2ad5:B 2024-11-07T12:43:07,484 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:43:07,484 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:43:07,484 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): 850d939a0dd41b63931f314702af2ad5/C is initiating minor compaction (all files) 2024-11-07T12:43:07,484 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 850d939a0dd41b63931f314702af2ad5/C in TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:07,484 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/ef5e053528fc4de794b3f37b3c697aa2, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/5df70c77ac1e499c87de84f6dd8652e4, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/d722000c169245bc987716f59b6c21b5] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp, totalSize=35.8 K 2024-11-07T12:43:07,485 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef5e053528fc4de794b3f37b3c697aa2, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1730983385987 2024-11-07T12:43:07,485 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5df70c77ac1e499c87de84f6dd8652e4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1730983386702 2024-11-07T12:43:07,504 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting d722000c169245bc987716f59b6c21b5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1730983387019 2024-11-07T12:43:07,516 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 850d939a0dd41b63931f314702af2ad5#C#compaction#470 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:43:07,517 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/841899e585b84216ae16585f18802c3f is 50, key is test_row_0/C:col10/1730983387019/Put/seqid=0 2024-11-07T12:43:07,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 850d939a0dd41b63931f314702af2ad5 2024-11-07T12:43:07,525 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 850d939a0dd41b63931f314702af2ad5 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-07T12:43:07,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=A 2024-11-07T12:43:07,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:07,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=B 2024-11-07T12:43:07,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:07,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=C 2024-11-07T12:43:07,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:07,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-07T12:43:07,539 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-07T12:43:07,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:07,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. as already flushing 2024-11-07T12:43:07,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:07,539 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:07,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:07,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:07,540 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/f2240a78a5d54d2e86f854ccc8b51c12 is 50, key is test_row_0/A:col10/1730983387523/Put/seqid=0 2024-11-07T12:43:07,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742384_1560 (size=12493) 2024-11-07T12:43:07,568 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/841899e585b84216ae16585f18802c3f as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/841899e585b84216ae16585f18802c3f 2024-11-07T12:43:07,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742385_1561 (size=14541) 2024-11-07T12:43:07,577 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/f2240a78a5d54d2e86f854ccc8b51c12 2024-11-07T12:43:07,580 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 850d939a0dd41b63931f314702af2ad5/C of 850d939a0dd41b63931f314702af2ad5 into 841899e585b84216ae16585f18802c3f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:43:07,580 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 850d939a0dd41b63931f314702af2ad5: 2024-11-07T12:43:07,580 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., storeName=850d939a0dd41b63931f314702af2ad5/C, priority=13, startTime=1730983387483; duration=0sec 2024-11-07T12:43:07,580 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:07,580 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 850d939a0dd41b63931f314702af2ad5:C 2024-11-07T12:43:07,594 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/d1990c70fd864669a4391d403fd8e3de is 50, key is test_row_0/B:col10/1730983387523/Put/seqid=0 2024-11-07T12:43:07,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742386_1562 (size=12151) 2024-11-07T12:43:07,609 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/d1990c70fd864669a4391d403fd8e3de 2024-11-07T12:43:07,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:07,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41722 deadline: 1730983447602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:07,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:07,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:07,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:07,612 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1730983387577, max timestamp=1730983387608Num uniques 859; 2024-11-07T12:43:07,612 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-07T12:43:07,615 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1730983387577, max timestamp=1730983387608Num uniques 859; 2024-11-07T12:43:07,615 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-07T12:43:07,615 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1730983387577, max timestamp=1730983387608Num uniques 859; 2024-11-07T12:43:07,615 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-07T12:43:07,623 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/2b6e6844916c46ceb0b2e690f073f2d0 is 50, key is test_row_0/C:col10/1730983387523/Put/seqid=0 2024-11-07T12:43:07,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742387_1563 (size=12151) 2024-11-07T12:43:07,691 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-07T12:43:07,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:07,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. as already flushing 2024-11-07T12:43:07,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:07,692 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:07,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:07,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:07,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:07,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1730983447685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:07,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:07,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41694 deadline: 1730983447703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:07,708 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:07,708 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:07,708 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:07,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:07,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:07,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:07,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:07,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41730 deadline: 1730983447705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:07,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:07,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:07,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:07,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:07,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41722 deadline: 1730983447721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:07,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:07,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:07,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:07,814 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/96088dd6609140428ac6f53fca1852c9 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/96088dd6609140428ac6f53fca1852c9 2024-11-07T12:43:07,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:07,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1730983447812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:07,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:07,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41694 deadline: 1730983447812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:07,818 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:07,818 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:07,818 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:07,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:07,819 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:07,819 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:07,819 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 850d939a0dd41b63931f314702af2ad5/B of 850d939a0dd41b63931f314702af2ad5 into 96088dd6609140428ac6f53fca1852c9(size=12.2 K), total size for store is 24.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:43:07,819 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 850d939a0dd41b63931f314702af2ad5: 2024-11-07T12:43:07,819 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., storeName=850d939a0dd41b63931f314702af2ad5/B, priority=13, startTime=1730983387344; duration=0sec 2024-11-07T12:43:07,819 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:07,819 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 850d939a0dd41b63931f314702af2ad5:B 2024-11-07T12:43:07,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:07,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41730 deadline: 1730983447832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:07,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:07,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:07,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:07,845 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-07T12:43:07,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:07,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. as already flushing 2024-11-07T12:43:07,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:07,845 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:07,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:07,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:07,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:07,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41722 deadline: 1730983447932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:07,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:07,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:07,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:07,998 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-07T12:43:07,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:07,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. as already flushing 2024-11-07T12:43:07,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:07,999 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:07,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:07,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:08,031 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/2b6e6844916c46ceb0b2e690f073f2d0 2024-11-07T12:43:08,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:08,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1730983448022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:08,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:08,032 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:08,032 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:08,033 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:08,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41694 deadline: 1730983448022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:08,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:08,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:08,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:08,035 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/f2240a78a5d54d2e86f854ccc8b51c12 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/f2240a78a5d54d2e86f854ccc8b51c12 2024-11-07T12:43:08,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-07T12:43:08,057 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/f2240a78a5d54d2e86f854ccc8b51c12, entries=200, sequenceid=203, filesize=14.2 K 2024-11-07T12:43:08,058 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/d1990c70fd864669a4391d403fd8e3de as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/d1990c70fd864669a4391d403fd8e3de 2024-11-07T12:43:08,063 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/d1990c70fd864669a4391d403fd8e3de, entries=150, sequenceid=203, filesize=11.9 K 2024-11-07T12:43:08,063 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/2b6e6844916c46ceb0b2e690f073f2d0 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/2b6e6844916c46ceb0b2e690f073f2d0 2024-11-07T12:43:08,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:08,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41730 deadline: 1730983448052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:08,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:08,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:08,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:08,067 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/2b6e6844916c46ceb0b2e690f073f2d0, entries=150, sequenceid=203, filesize=11.9 K 2024-11-07T12:43:08,068 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 850d939a0dd41b63931f314702af2ad5 in 543ms, sequenceid=203, compaction requested=true 2024-11-07T12:43:08,068 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 850d939a0dd41b63931f314702af2ad5: 2024-11-07T12:43:08,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 850d939a0dd41b63931f314702af2ad5:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:43:08,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:08,069 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:43:08,069 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:43:08,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 850d939a0dd41b63931f314702af2ad5:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:43:08,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:08,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 850d939a0dd41b63931f314702af2ad5:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:43:08,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:43:08,070 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41541 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:43:08,070 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): 850d939a0dd41b63931f314702af2ad5/A is initiating minor compaction (all files) 2024-11-07T12:43:08,070 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:43:08,070 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 850d939a0dd41b63931f314702af2ad5/A in TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:08,070 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 850d939a0dd41b63931f314702af2ad5/B is initiating minor compaction (all files) 2024-11-07T12:43:08,070 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 850d939a0dd41b63931f314702af2ad5/B in TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:08,070 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/c1e4851b3abb42159c2fb0eb7e7729b7, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/721f5882475c452ba689e2dd4a4736ef, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/f2240a78a5d54d2e86f854ccc8b51c12] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp, totalSize=40.6 K 2024-11-07T12:43:08,070 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/96088dd6609140428ac6f53fca1852c9, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/dad736474d52401298f14efec800557b, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/d1990c70fd864669a4391d403fd8e3de] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp, totalSize=35.9 K 2024-11-07T12:43:08,070 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1e4851b3abb42159c2fb0eb7e7729b7, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1730983386702 2024-11-07T12:43:08,070 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 96088dd6609140428ac6f53fca1852c9, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1730983386702 2024-11-07T12:43:08,071 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 721f5882475c452ba689e2dd4a4736ef, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1730983387019 2024-11-07T12:43:08,071 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting dad736474d52401298f14efec800557b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1730983387019 2024-11-07T12:43:08,071 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2240a78a5d54d2e86f854ccc8b51c12, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1730983387373 2024-11-07T12:43:08,071 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting d1990c70fd864669a4391d403fd8e3de, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1730983387392 2024-11-07T12:43:08,080 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 850d939a0dd41b63931f314702af2ad5#B#compaction#474 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:43:08,081 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/12b8e69d23e945cea852b93c5efc3d0e is 50, key is test_row_0/B:col10/1730983387523/Put/seqid=0 2024-11-07T12:43:08,083 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 850d939a0dd41b63931f314702af2ad5#A#compaction#475 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:43:08,083 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/caaa87115cbd49e8b35b237f1d3fe9a0 is 50, key is test_row_0/A:col10/1730983387523/Put/seqid=0 2024-11-07T12:43:08,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742388_1564 (size=12561) 2024-11-07T12:43:08,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742389_1565 (size=12561) 2024-11-07T12:43:08,101 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/12b8e69d23e945cea852b93c5efc3d0e as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/12b8e69d23e945cea852b93c5efc3d0e 2024-11-07T12:43:08,101 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/caaa87115cbd49e8b35b237f1d3fe9a0 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/caaa87115cbd49e8b35b237f1d3fe9a0 2024-11-07T12:43:08,107 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 850d939a0dd41b63931f314702af2ad5/B of 850d939a0dd41b63931f314702af2ad5 into 12b8e69d23e945cea852b93c5efc3d0e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:43:08,107 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 850d939a0dd41b63931f314702af2ad5/A of 850d939a0dd41b63931f314702af2ad5 into caaa87115cbd49e8b35b237f1d3fe9a0(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:43:08,107 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 850d939a0dd41b63931f314702af2ad5: 2024-11-07T12:43:08,107 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., storeName=850d939a0dd41b63931f314702af2ad5/A, priority=13, startTime=1730983388069; duration=0sec 2024-11-07T12:43:08,107 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 850d939a0dd41b63931f314702af2ad5: 2024-11-07T12:43:08,107 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:43:08,107 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., storeName=850d939a0dd41b63931f314702af2ad5/B, priority=13, startTime=1730983388069; duration=0sec 2024-11-07T12:43:08,107 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 850d939a0dd41b63931f314702af2ad5:A 2024-11-07T12:43:08,107 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-07T12:43:08,107 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:08,107 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 850d939a0dd41b63931f314702af2ad5:B 2024-11-07T12:43:08,108 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-07T12:43:08,108 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-07T12:43:08,108 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. because compaction request was cancelled 2024-11-07T12:43:08,109 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 850d939a0dd41b63931f314702af2ad5:C 2024-11-07T12:43:08,152 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-07T12:43:08,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:08,152 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 850d939a0dd41b63931f314702af2ad5 3/3 column families, dataSize=134.18 KB heapSize=370.92 KB 2024-11-07T12:43:08,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=A 2024-11-07T12:43:08,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:43:08,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=B 2024-11-07T12:43:08,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:43:08,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=C 2024-11-07T12:43:08,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:43:08,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/78beb4d69b6b4ef392c107ba82dd5b66 is 50, key is test_row_0/A:col10/1730983387677/Put/seqid=0 2024-11-07T12:43:08,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742390_1566 (size=12151) 2024-11-07T12:43:08,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 850d939a0dd41b63931f314702af2ad5 2024-11-07T12:43:08,259 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. as already flushing 2024-11-07T12:43:08,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:08,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41722 deadline: 1730983448309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:08,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:08,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:08,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:08,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:08,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41752 deadline: 1730983448310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:08,325 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:08,325 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:08,325 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:08,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:08,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1730983448341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:08,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:08,348 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:08,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41694 deadline: 1730983448341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:08,348 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:08,348 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:08,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:08,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:08,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:08,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:08,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41730 deadline: 1730983448372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:08,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:08,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:08,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:08,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:08,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41722 deadline: 1730983448431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:08,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:08,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:08,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:08,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41752 deadline: 1730983448432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:08,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:08,440 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:08,440 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:08,440 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:08,569 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/78beb4d69b6b4ef392c107ba82dd5b66 2024-11-07T12:43:08,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/2a1d7360b89445f98d47e496c9d1fa61 is 50, key is test_row_0/B:col10/1730983387677/Put/seqid=0 2024-11-07T12:43:08,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742391_1567 (size=12151) 2024-11-07T12:43:08,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:08,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41722 deadline: 1730983448652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:08,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:08,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41752 deadline: 1730983448652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:08,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:08,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:08,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:08,661 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:08,661 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:08,661 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:08,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:08,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1730983448851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:08,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:08,867 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:08,867 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:08,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41694 deadline: 1730983448851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:08,867 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:08,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:08,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:08,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:08,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:08,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41730 deadline: 1730983448892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:08,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:08,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:08,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:08,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:08,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41752 deadline: 1730983448972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:08,977 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:08,977 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:08,977 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:08,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:08,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41722 deadline: 1730983448972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:08,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:08,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:08,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:08,984 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/2a1d7360b89445f98d47e496c9d1fa61 2024-11-07T12:43:09,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/96adca44c84f49a390335a6d6c4eb5c7 is 50, key is test_row_0/C:col10/1730983387677/Put/seqid=0 2024-11-07T12:43:09,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742392_1568 (size=12151) 2024-11-07T12:43:09,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-07T12:43:09,425 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/96adca44c84f49a390335a6d6c4eb5c7 2024-11-07T12:43:09,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/78beb4d69b6b4ef392c107ba82dd5b66 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/78beb4d69b6b4ef392c107ba82dd5b66 2024-11-07T12:43:09,433 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/78beb4d69b6b4ef392c107ba82dd5b66, entries=150, sequenceid=241, filesize=11.9 K 2024-11-07T12:43:09,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/2a1d7360b89445f98d47e496c9d1fa61 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/2a1d7360b89445f98d47e496c9d1fa61 2024-11-07T12:43:09,437 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/2a1d7360b89445f98d47e496c9d1fa61, entries=150, sequenceid=241, filesize=11.9 K 2024-11-07T12:43:09,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/96adca44c84f49a390335a6d6c4eb5c7 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/96adca44c84f49a390335a6d6c4eb5c7 2024-11-07T12:43:09,444 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/96adca44c84f49a390335a6d6c4eb5c7, entries=150, sequenceid=241, filesize=11.9 K 2024-11-07T12:43:09,445 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~370.88 KB/379776, currentSize=53.67 KB/54960 for 850d939a0dd41b63931f314702af2ad5 in 1293ms, sequenceid=241, compaction requested=true 2024-11-07T12:43:09,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 850d939a0dd41b63931f314702af2ad5: 2024-11-07T12:43:09,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:09,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-07T12:43:09,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-07T12:43:09,448 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-07T12:43:09,448 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5310 sec 2024-11-07T12:43:09,451 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 2.5460 sec 2024-11-07T12:43:09,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 850d939a0dd41b63931f314702af2ad5 2024-11-07T12:43:09,495 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 850d939a0dd41b63931f314702af2ad5 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-07T12:43:09,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=A 2024-11-07T12:43:09,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:09,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=B 2024-11-07T12:43:09,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:09,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=C 2024-11-07T12:43:09,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:09,500 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/cc2f0a28846d47b3b0c1a899601689c0 is 50, key is test_row_1/A:col10/1730983389493/Put/seqid=0 2024-11-07T12:43:09,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742393_1569 (size=12147) 2024-11-07T12:43:09,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:09,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41722 deadline: 1730983449575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:09,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:09,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:09,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:09,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:09,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41752 deadline: 1730983449577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:09,588 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:09,588 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:09,588 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:09,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:09,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41722 deadline: 1730983449692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:09,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:09,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41752 deadline: 1730983449692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:09,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:09,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:09,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:09,700 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:09,700 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:09,700 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:09,718 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-07T12:43:09,718 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-07T12:43:09,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:09,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1730983449872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:09,879 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to db9ad1cb6cf9,34293,1730983260217 for 'test_row_2' in 850d939a0dd41b63931f314702af2ad5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:43:09,879 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:09,879 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:09,879 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:09,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:09,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41694 deadline: 1730983449881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:09,889 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to db9ad1cb6cf9,34293,1730983260217 for 'test_row_2' in 850d939a0dd41b63931f314702af2ad5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:43:09,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:09,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:09,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:09,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:09,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41752 deadline: 1730983449902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:09,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:09,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41722 deadline: 1730983449902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:09,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:09,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:09,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:09,914 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:09,914 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:09,914 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:09,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:09,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41730 deadline: 1730983449911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:09,917 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to db9ad1cb6cf9,34293,1730983260217 for 'test_row_0' in 850d939a0dd41b63931f314702af2ad5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2212 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:43:09,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:09,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:09,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:09,922 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/cc2f0a28846d47b3b0c1a899601689c0 2024-11-07T12:43:09,932 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/aa990acda5fe48408886670984010f6a is 50, key is test_row_1/B:col10/1730983389493/Put/seqid=0 2024-11-07T12:43:09,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742394_1570 (size=9757) 2024-11-07T12:43:09,939 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/aa990acda5fe48408886670984010f6a 2024-11-07T12:43:09,970 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/b7edaefc324947deade9146a2a2f153d is 50, key is test_row_1/C:col10/1730983389493/Put/seqid=0 2024-11-07T12:43:09,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742395_1571 (size=9757) 2024-11-07T12:43:10,238 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:10,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41752 deadline: 1730983450222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:10,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:10,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41722 deadline: 1730983450222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:10,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:10,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:10,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:10,240 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:10,241 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:10,241 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:10,385 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/b7edaefc324947deade9146a2a2f153d 2024-11-07T12:43:10,397 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/cc2f0a28846d47b3b0c1a899601689c0 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/cc2f0a28846d47b3b0c1a899601689c0 2024-11-07T12:43:10,416 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/cc2f0a28846d47b3b0c1a899601689c0, entries=150, sequenceid=253, filesize=11.9 K 2024-11-07T12:43:10,417 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/aa990acda5fe48408886670984010f6a as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/aa990acda5fe48408886670984010f6a 2024-11-07T12:43:10,422 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/aa990acda5fe48408886670984010f6a, entries=100, sequenceid=253, filesize=9.5 K 2024-11-07T12:43:10,423 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/b7edaefc324947deade9146a2a2f153d as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/b7edaefc324947deade9146a2a2f153d 2024-11-07T12:43:10,428 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/b7edaefc324947deade9146a2a2f153d, entries=100, sequenceid=253, filesize=9.5 K 2024-11-07T12:43:10,430 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 850d939a0dd41b63931f314702af2ad5 in 936ms, sequenceid=253, compaction requested=true 2024-11-07T12:43:10,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 850d939a0dd41b63931f314702af2ad5: 2024-11-07T12:43:10,430 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:43:10,434 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36859 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:43:10,434 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): 850d939a0dd41b63931f314702af2ad5/A is initiating minor compaction (all files) 2024-11-07T12:43:10,434 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 850d939a0dd41b63931f314702af2ad5/A in TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:10,434 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/caaa87115cbd49e8b35b237f1d3fe9a0, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/78beb4d69b6b4ef392c107ba82dd5b66, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/cc2f0a28846d47b3b0c1a899601689c0] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp, totalSize=36.0 K 2024-11-07T12:43:10,435 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting caaa87115cbd49e8b35b237f1d3fe9a0, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1730983387392 2024-11-07T12:43:10,436 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 78beb4d69b6b4ef392c107ba82dd5b66, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1730983387677 2024-11-07T12:43:10,436 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc2f0a28846d47b3b0c1a899601689c0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1730983388305 2024-11-07T12:43:10,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 850d939a0dd41b63931f314702af2ad5:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:43:10,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:10,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 850d939a0dd41b63931f314702af2ad5:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:43:10,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:10,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 850d939a0dd41b63931f314702af2ad5:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:43:10,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:43:10,440 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:43:10,441 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34469 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:43:10,441 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 850d939a0dd41b63931f314702af2ad5/B is initiating minor compaction (all files) 2024-11-07T12:43:10,441 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 850d939a0dd41b63931f314702af2ad5/B in TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:10,441 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/12b8e69d23e945cea852b93c5efc3d0e, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/2a1d7360b89445f98d47e496c9d1fa61, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/aa990acda5fe48408886670984010f6a] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp, totalSize=33.7 K 2024-11-07T12:43:10,448 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 12b8e69d23e945cea852b93c5efc3d0e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1730983387392 2024-11-07T12:43:10,449 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a1d7360b89445f98d47e496c9d1fa61, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1730983387677 2024-11-07T12:43:10,451 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting aa990acda5fe48408886670984010f6a, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1730983388305 2024-11-07T12:43:10,455 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 850d939a0dd41b63931f314702af2ad5#A#compaction#482 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:43:10,456 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/b3486332d47c4c89b240d2cd297207e7 is 50, key is test_row_0/A:col10/1730983387677/Put/seqid=0 2024-11-07T12:43:10,471 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 850d939a0dd41b63931f314702af2ad5#B#compaction#483 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:43:10,471 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/d3cf1868acbe44f29774553dcb68064c is 50, key is test_row_0/B:col10/1730983387677/Put/seqid=0 2024-11-07T12:43:10,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742396_1572 (size=12663) 2024-11-07T12:43:10,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742397_1573 (size=12663) 2024-11-07T12:43:10,535 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/b3486332d47c4c89b240d2cd297207e7 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/b3486332d47c4c89b240d2cd297207e7 2024-11-07T12:43:10,541 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 850d939a0dd41b63931f314702af2ad5/A of 850d939a0dd41b63931f314702af2ad5 into b3486332d47c4c89b240d2cd297207e7(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:43:10,541 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 850d939a0dd41b63931f314702af2ad5: 2024-11-07T12:43:10,541 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., storeName=850d939a0dd41b63931f314702af2ad5/A, priority=13, startTime=1730983390430; duration=0sec 2024-11-07T12:43:10,541 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:43:10,541 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 850d939a0dd41b63931f314702af2ad5:A 2024-11-07T12:43:10,541 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:43:10,549 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46552 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:43:10,549 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): 850d939a0dd41b63931f314702af2ad5/C is initiating minor compaction (all files) 2024-11-07T12:43:10,549 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 850d939a0dd41b63931f314702af2ad5/C in TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:10,549 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/841899e585b84216ae16585f18802c3f, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/2b6e6844916c46ceb0b2e690f073f2d0, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/96adca44c84f49a390335a6d6c4eb5c7, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/b7edaefc324947deade9146a2a2f153d] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp, totalSize=45.5 K 2024-11-07T12:43:10,549 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 841899e585b84216ae16585f18802c3f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1730983387019 2024-11-07T12:43:10,550 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b6e6844916c46ceb0b2e690f073f2d0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1730983387392 2024-11-07T12:43:10,551 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96adca44c84f49a390335a6d6c4eb5c7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1730983387677 2024-11-07T12:43:10,551 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7edaefc324947deade9146a2a2f153d, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1730983388305 2024-11-07T12:43:10,560 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 850d939a0dd41b63931f314702af2ad5#C#compaction#484 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:43:10,561 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/51bca35d4f0f4b548ee81f7971358962 is 50, key is test_row_0/C:col10/1730983387677/Put/seqid=0 2024-11-07T12:43:10,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742398_1574 (size=12629) 2024-11-07T12:43:10,594 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/51bca35d4f0f4b548ee81f7971358962 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/51bca35d4f0f4b548ee81f7971358962 2024-11-07T12:43:10,602 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 850d939a0dd41b63931f314702af2ad5/C of 850d939a0dd41b63931f314702af2ad5 into 51bca35d4f0f4b548ee81f7971358962(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:43:10,603 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 850d939a0dd41b63931f314702af2ad5: 2024-11-07T12:43:10,603 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., storeName=850d939a0dd41b63931f314702af2ad5/C, priority=12, startTime=1730983390439; duration=0sec 2024-11-07T12:43:10,603 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:10,604 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 850d939a0dd41b63931f314702af2ad5:C 2024-11-07T12:43:10,760 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1730983389528, max timestamp=1730983390757Num uniques 859; 2024-11-07T12:43:10,761 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-07T12:43:10,765 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1730983389528, max timestamp=1730983390757Num uniques 859; 2024-11-07T12:43:10,765 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-07T12:43:10,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 850d939a0dd41b63931f314702af2ad5 2024-11-07T12:43:10,766 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 850d939a0dd41b63931f314702af2ad5 3/3 column families, dataSize=58.14 KB heapSize=164.17 KB 2024-11-07T12:43:10,766 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1730983389528, max timestamp=1730983390763Num uniques 859; 2024-11-07T12:43:10,766 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-07T12:43:10,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=A 2024-11-07T12:43:10,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:43:10,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=B 2024-11-07T12:43:10,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:43:10,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=C 2024-11-07T12:43:10,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:43:10,774 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/cd34d10eb9ff4ec097055a98c1bc0438 is 50, key is test_row_0/A:col10/1730983389528/Put/seqid=0 2024-11-07T12:43:10,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742399_1575 (size=17181) 2024-11-07T12:43:10,790 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/cd34d10eb9ff4ec097055a98c1bc0438 2024-11-07T12:43:10,796 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/f84ede0489a24be1993917ddb87564ff is 50, key is test_row_0/B:col10/1730983389528/Put/seqid=0 2024-11-07T12:43:10,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742400_1576 (size=12301) 2024-11-07T12:43:10,891 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1730983390859, max timestamp=1730983390887Num uniques 859; 2024-11-07T12:43:10,891 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1730983390859, max timestamp=1730983390887Num uniques 859; 2024-11-07T12:43:10,891 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-07T12:43:10,891 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-07T12:43:10,893 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1730983390859, max timestamp=1730983390887Num uniques 859; 2024-11-07T12:43:10,893 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-07T12:43:10,928 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/d3cf1868acbe44f29774553dcb68064c as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/d3cf1868acbe44f29774553dcb68064c 2024-11-07T12:43:10,932 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 850d939a0dd41b63931f314702af2ad5/B of 850d939a0dd41b63931f314702af2ad5 into d3cf1868acbe44f29774553dcb68064c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:43:10,932 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 850d939a0dd41b63931f314702af2ad5: 2024-11-07T12:43:10,932 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., storeName=850d939a0dd41b63931f314702af2ad5/B, priority=13, startTime=1730983390439; duration=0sec 2024-11-07T12:43:10,932 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:10,932 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 850d939a0dd41b63931f314702af2ad5:B 2024-11-07T12:43:10,989 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1730983390924, max timestamp=1730983390984Num uniques 1068; 2024-11-07T12:43:10,989 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-11-07T12:43:10,989 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1730983390924, max timestamp=1730983390984Num uniques 1068; 2024-11-07T12:43:10,989 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-11-07T12:43:10,990 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1730983390924, max timestamp=1730983390984Num uniques 1068; 2024-11-07T12:43:10,990 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-11-07T12:43:11,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-07T12:43:11,071 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:43:11,072 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:43:11,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-07T12:43:11,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-07T12:43:11,073 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:43:11,074 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:43:11,074 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:43:11,102 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1730983391021, max timestamp=1730983391096Num uniques 1127; 2024-11-07T12:43:11,102 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1730983391021, max timestamp=1730983391096Num uniques 1127; 2024-11-07T12:43:11,102 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=10392, suffix heap size=145176,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=227 2024-11-07T12:43:11,102 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=10392, suffix heap size=145176,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=227 2024-11-07T12:43:11,104 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1730983391021, max timestamp=1730983391096Num uniques 1127; 2024-11-07T12:43:11,104 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=10392, suffix heap size=145176,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=227 2024-11-07T12:43:11,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-07T12:43:11,198 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1730983391151, max timestamp=1730983391191Num uniques 1086; 2024-11-07T12:43:11,198 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-07T12:43:11,199 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1730983391151, max timestamp=1730983391191Num uniques 1086; 2024-11-07T12:43:11,198 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1730983391151, max timestamp=1730983391191Num uniques 1086; 2024-11-07T12:43:11,199 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-07T12:43:11,200 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-07T12:43:11,213 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/f84ede0489a24be1993917ddb87564ff 2024-11-07T12:43:11,222 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/e14e8d11df9a40a99bb4253f9d234108 is 50, key is test_row_0/C:col10/1730983389528/Put/seqid=0 2024-11-07T12:43:11,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742401_1577 (size=12301) 2024-11-07T12:43:11,226 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/e14e8d11df9a40a99bb4253f9d234108 2024-11-07T12:43:11,227 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-07T12:43:11,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:11,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. as already flushing 2024-11-07T12:43:11,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:11,227 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:11,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:11,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:11,236 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/cd34d10eb9ff4ec097055a98c1bc0438 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/cd34d10eb9ff4ec097055a98c1bc0438 2024-11-07T12:43:11,239 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/cd34d10eb9ff4ec097055a98c1bc0438, entries=250, sequenceid=277, filesize=16.8 K 2024-11-07T12:43:11,240 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/f84ede0489a24be1993917ddb87564ff as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/f84ede0489a24be1993917ddb87564ff 2024-11-07T12:43:11,243 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/f84ede0489a24be1993917ddb87564ff, entries=150, sequenceid=277, filesize=12.0 K 2024-11-07T12:43:11,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/e14e8d11df9a40a99bb4253f9d234108 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/e14e8d11df9a40a99bb4253f9d234108 2024-11-07T12:43:11,248 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/e14e8d11df9a40a99bb4253f9d234108, entries=150, sequenceid=277, filesize=12.0 K 2024-11-07T12:43:11,251 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~86.11 KB/88176, currentSize=87.22 KB/89310 for 850d939a0dd41b63931f314702af2ad5 in 485ms, sequenceid=277, compaction requested=false 2024-11-07T12:43:11,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 850d939a0dd41b63931f314702af2ad5: 2024-11-07T12:43:11,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 850d939a0dd41b63931f314702af2ad5 2024-11-07T12:43:11,256 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 850d939a0dd41b63931f314702af2ad5 3/3 column families, dataSize=93.93 KB heapSize=258.23 KB 2024-11-07T12:43:11,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=A 2024-11-07T12:43:11,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:43:11,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=B 2024-11-07T12:43:11,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:43:11,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=C 2024-11-07T12:43:11,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:43:11,262 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/a14ab44472144ecfa8b2d466c9612f98 is 50, key is test_row_0/A:col10/1730983391254/Put/seqid=0 2024-11-07T12:43:11,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742402_1578 (size=17181) 2024-11-07T12:43:11,287 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/a14ab44472144ecfa8b2d466c9612f98 2024-11-07T12:43:11,312 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/313fdbb6092a4b7bb8e9a8d21b1d1514 is 50, key is test_row_0/B:col10/1730983391254/Put/seqid=0 2024-11-07T12:43:11,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742403_1579 (size=12301) 2024-11-07T12:43:11,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:11,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41752 deadline: 1730983451344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:11,365 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:11,365 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:11,365 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:11,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:11,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41722 deadline: 1730983451361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:11,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:11,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:11,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:11,381 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-07T12:43:11,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:11,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. as already flushing 2024-11-07T12:43:11,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:11,381 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:11,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:11,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:11,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-07T12:43:11,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:11,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41752 deadline: 1730983451472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:11,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:11,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41722 deadline: 1730983451472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:11,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:11,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:11,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:11,487 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:11,488 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:11,488 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:11,533 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-07T12:43:11,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:11,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. as already flushing 2024-11-07T12:43:11,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:11,534 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:11,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:11,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:11,687 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-07T12:43:11,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:11,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. as already flushing 2024-11-07T12:43:11,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:11,688 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:11,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:11,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:11,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-07T12:43:11,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:11,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41722 deadline: 1730983451691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:11,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:11,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41752 deadline: 1730983451691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:11,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:11,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:11,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:11,704 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:11,704 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:11,704 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:11,720 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/313fdbb6092a4b7bb8e9a8d21b1d1514 2024-11-07T12:43:11,728 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/f779f664483745c197cca3200851c245 is 50, key is test_row_0/C:col10/1730983391254/Put/seqid=0 2024-11-07T12:43:11,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742404_1580 (size=12301) 2024-11-07T12:43:11,841 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-07T12:43:11,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:11,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. as already flushing 2024-11-07T12:43:11,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:11,841 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:11,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:11,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:11,913 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:11,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1730983451901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:11,914 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to db9ad1cb6cf9,34293,1730983260217 for 'test_row_2' in 850d939a0dd41b63931f314702af2ad5 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4230 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:43:11,915 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:11,915 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:11,915 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:11,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:11,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41694 deadline: 1730983451911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:11,921 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to db9ad1cb6cf9,34293,1730983260217 for 'test_row_2' in 850d939a0dd41b63931f314702af2ad5 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4219 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:43:11,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:11,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:11,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:11,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:11,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41730 deadline: 1730983451932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:11,941 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to db9ad1cb6cf9,34293,1730983260217 for 'test_row_0' in 850d939a0dd41b63931f314702af2ad5 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4236 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:43:11,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:11,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:11,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:11,994 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-07T12:43:11,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:11,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. as already flushing 2024-11-07T12:43:11,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:11,994 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:11,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:11,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:12,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:12,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41722 deadline: 1730983452012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:12,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:12,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41752 deadline: 1730983452012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:12,023 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:12,023 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:12,023 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:12,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:12,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:12,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:12,132 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/f779f664483745c197cca3200851c245 2024-11-07T12:43:12,136 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/a14ab44472144ecfa8b2d466c9612f98 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/a14ab44472144ecfa8b2d466c9612f98 2024-11-07T12:43:12,139 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/a14ab44472144ecfa8b2d466c9612f98, entries=250, sequenceid=360, filesize=16.8 K 2024-11-07T12:43:12,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/313fdbb6092a4b7bb8e9a8d21b1d1514 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/313fdbb6092a4b7bb8e9a8d21b1d1514 2024-11-07T12:43:12,143 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/313fdbb6092a4b7bb8e9a8d21b1d1514, entries=150, sequenceid=360, filesize=12.0 K 2024-11-07T12:43:12,144 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/f779f664483745c197cca3200851c245 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/f779f664483745c197cca3200851c245 2024-11-07T12:43:12,149 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-07T12:43:12,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:12,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. as already flushing 2024-11-07T12:43:12,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:12,149 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:12,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:12,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:12,151 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/f779f664483745c197cca3200851c245, entries=150, sequenceid=360, filesize=12.0 K 2024-11-07T12:43:12,152 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~278.11 KB/284784, currentSize=80.51 KB/82440 for 850d939a0dd41b63931f314702af2ad5 in 895ms, sequenceid=360, compaction requested=true 2024-11-07T12:43:12,152 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 850d939a0dd41b63931f314702af2ad5: 2024-11-07T12:43:12,152 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 850d939a0dd41b63931f314702af2ad5:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:43:12,152 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:12,152 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:43:12,152 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 850d939a0dd41b63931f314702af2ad5:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:43:12,152 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:12,152 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 850d939a0dd41b63931f314702af2ad5:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:43:12,152 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:43:12,152 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:43:12,153 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37265 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:43:12,153 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 47025 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:43:12,154 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 850d939a0dd41b63931f314702af2ad5/B is initiating minor compaction (all files) 2024-11-07T12:43:12,154 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): 850d939a0dd41b63931f314702af2ad5/A is initiating minor compaction (all files) 2024-11-07T12:43:12,154 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 850d939a0dd41b63931f314702af2ad5/B in TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:12,154 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 850d939a0dd41b63931f314702af2ad5/A in TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:12,154 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/b3486332d47c4c89b240d2cd297207e7, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/cd34d10eb9ff4ec097055a98c1bc0438, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/a14ab44472144ecfa8b2d466c9612f98] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp, totalSize=45.9 K 2024-11-07T12:43:12,154 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/d3cf1868acbe44f29774553dcb68064c, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/f84ede0489a24be1993917ddb87564ff, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/313fdbb6092a4b7bb8e9a8d21b1d1514] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp, totalSize=36.4 K 2024-11-07T12:43:12,154 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3486332d47c4c89b240d2cd297207e7, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1730983387677 2024-11-07T12:43:12,154 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting d3cf1868acbe44f29774553dcb68064c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1730983387677 2024-11-07T12:43:12,154 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd34d10eb9ff4ec097055a98c1bc0438, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1730983389528 2024-11-07T12:43:12,154 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting f84ede0489a24be1993917ddb87564ff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1730983389528 2024-11-07T12:43:12,155 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting a14ab44472144ecfa8b2d466c9612f98, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1730983391229 2024-11-07T12:43:12,155 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 313fdbb6092a4b7bb8e9a8d21b1d1514, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1730983391229 2024-11-07T12:43:12,167 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 850d939a0dd41b63931f314702af2ad5#B#compaction#491 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:43:12,168 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/dd11685f640c452e91fbc824a8fb70e7 is 50, key is test_row_0/B:col10/1730983391254/Put/seqid=0 2024-11-07T12:43:12,177 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 850d939a0dd41b63931f314702af2ad5#A#compaction#492 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:43:12,177 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/1490b50bf5cf4b7d875305722ec3a696 is 50, key is test_row_0/A:col10/1730983391254/Put/seqid=0 2024-11-07T12:43:12,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742405_1581 (size=12915) 2024-11-07T12:43:12,199 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/dd11685f640c452e91fbc824a8fb70e7 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/dd11685f640c452e91fbc824a8fb70e7 2024-11-07T12:43:12,205 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 850d939a0dd41b63931f314702af2ad5/B of 850d939a0dd41b63931f314702af2ad5 into dd11685f640c452e91fbc824a8fb70e7(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:43:12,205 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 850d939a0dd41b63931f314702af2ad5: 2024-11-07T12:43:12,205 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., storeName=850d939a0dd41b63931f314702af2ad5/B, priority=13, startTime=1730983392152; duration=0sec 2024-11-07T12:43:12,205 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:43:12,205 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 850d939a0dd41b63931f314702af2ad5:B 2024-11-07T12:43:12,205 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:43:12,206 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37231 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:43:12,206 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 850d939a0dd41b63931f314702af2ad5/C is initiating minor compaction (all files) 2024-11-07T12:43:12,206 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 850d939a0dd41b63931f314702af2ad5/C in TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:12,206 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/51bca35d4f0f4b548ee81f7971358962, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/e14e8d11df9a40a99bb4253f9d234108, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/f779f664483745c197cca3200851c245] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp, totalSize=36.4 K 2024-11-07T12:43:12,207 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 51bca35d4f0f4b548ee81f7971358962, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1730983387677 2024-11-07T12:43:12,207 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting e14e8d11df9a40a99bb4253f9d234108, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1730983389528 2024-11-07T12:43:12,208 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting f779f664483745c197cca3200851c245, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1730983391229 2024-11-07T12:43:12,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-07T12:43:12,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742406_1582 (size=12915) 2024-11-07T12:43:12,223 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 850d939a0dd41b63931f314702af2ad5#C#compaction#493 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:43:12,223 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/d4b3de077bcc472ea350170dccbf424d is 50, key is test_row_0/C:col10/1730983391254/Put/seqid=0 2024-11-07T12:43:12,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742407_1583 (size=12881) 2024-11-07T12:43:12,251 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/d4b3de077bcc472ea350170dccbf424d as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/d4b3de077bcc472ea350170dccbf424d 2024-11-07T12:43:12,254 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 850d939a0dd41b63931f314702af2ad5/C of 850d939a0dd41b63931f314702af2ad5 into d4b3de077bcc472ea350170dccbf424d(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:43:12,254 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 850d939a0dd41b63931f314702af2ad5: 2024-11-07T12:43:12,254 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., storeName=850d939a0dd41b63931f314702af2ad5/C, priority=13, startTime=1730983392152; duration=0sec 2024-11-07T12:43:12,254 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:12,255 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 850d939a0dd41b63931f314702af2ad5:C 2024-11-07T12:43:12,302 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-07T12:43:12,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:12,303 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 850d939a0dd41b63931f314702af2ad5 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-07T12:43:12,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=A 2024-11-07T12:43:12,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:12,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=B 2024-11-07T12:43:12,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:12,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=C 2024-11-07T12:43:12,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:12,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/024d3bac75f44ff29b76b70530be07ac is 50, key is test_row_0/A:col10/1730983391335/Put/seqid=0 2024-11-07T12:43:12,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742408_1584 (size=12301) 2024-11-07T12:43:12,318 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/024d3bac75f44ff29b76b70530be07ac 2024-11-07T12:43:12,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/22188d97e5f549d7ad395e64100f7223 is 50, key is test_row_0/B:col10/1730983391335/Put/seqid=0 2024-11-07T12:43:12,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742409_1585 (size=12301) 2024-11-07T12:43:12,338 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/22188d97e5f549d7ad395e64100f7223 2024-11-07T12:43:12,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/7d7f59f69c6344b8b2334b608b9b05e1 is 50, key is test_row_0/C:col10/1730983391335/Put/seqid=0 2024-11-07T12:43:12,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742410_1586 (size=12301) 2024-11-07T12:43:12,362 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/7d7f59f69c6344b8b2334b608b9b05e1 2024-11-07T12:43:12,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/024d3bac75f44ff29b76b70530be07ac as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/024d3bac75f44ff29b76b70530be07ac 2024-11-07T12:43:12,386 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/024d3bac75f44ff29b76b70530be07ac, entries=150, sequenceid=377, filesize=12.0 K 2024-11-07T12:43:12,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/22188d97e5f549d7ad395e64100f7223 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/22188d97e5f549d7ad395e64100f7223 2024-11-07T12:43:12,394 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/22188d97e5f549d7ad395e64100f7223, entries=150, sequenceid=377, filesize=12.0 K 2024-11-07T12:43:12,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/7d7f59f69c6344b8b2334b608b9b05e1 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/7d7f59f69c6344b8b2334b608b9b05e1 2024-11-07T12:43:12,399 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/7d7f59f69c6344b8b2334b608b9b05e1, entries=150, sequenceid=377, filesize=12.0 K 2024-11-07T12:43:12,400 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=0 B/0 for 850d939a0dd41b63931f314702af2ad5 in 97ms, sequenceid=377, compaction requested=false 2024-11-07T12:43:12,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 850d939a0dd41b63931f314702af2ad5: 2024-11-07T12:43:12,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:12,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-07T12:43:12,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-07T12:43:12,406 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-07T12:43:12,407 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3280 sec 2024-11-07T12:43:12,409 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 1.3350 sec 2024-11-07T12:43:12,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 850d939a0dd41b63931f314702af2ad5 2024-11-07T12:43:12,555 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 850d939a0dd41b63931f314702af2ad5 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-07T12:43:12,555 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=A 2024-11-07T12:43:12,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:12,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=B 2024-11-07T12:43:12,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:12,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=C 2024-11-07T12:43:12,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:12,561 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/2c8ffd18227e47b0a36e3282e43fb737 is 50, key is test_row_0/A:col10/1730983392554/Put/seqid=0 2024-11-07T12:43:12,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742411_1587 (size=14741) 2024-11-07T12:43:12,588 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/2c8ffd18227e47b0a36e3282e43fb737 2024-11-07T12:43:12,596 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/c748e44bc30a451b9e823b6b91ca4858 is 50, key is test_row_0/B:col10/1730983392554/Put/seqid=0 2024-11-07T12:43:12,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742412_1588 (size=12301) 2024-11-07T12:43:12,607 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/c748e44bc30a451b9e823b6b91ca4858 2024-11-07T12:43:12,616 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/c7f9c8fe7b194819871450c0d2bcaeef is 50, key is test_row_0/C:col10/1730983392554/Put/seqid=0 2024-11-07T12:43:12,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742413_1589 (size=12301) 2024-11-07T12:43:12,633 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/c7f9c8fe7b194819871450c0d2bcaeef 2024-11-07T12:43:12,636 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/1490b50bf5cf4b7d875305722ec3a696 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/1490b50bf5cf4b7d875305722ec3a696 2024-11-07T12:43:12,639 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/2c8ffd18227e47b0a36e3282e43fb737 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/2c8ffd18227e47b0a36e3282e43fb737 2024-11-07T12:43:12,653 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/2c8ffd18227e47b0a36e3282e43fb737, entries=200, sequenceid=387, filesize=14.4 K 2024-11-07T12:43:12,657 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 850d939a0dd41b63931f314702af2ad5/A of 850d939a0dd41b63931f314702af2ad5 into 1490b50bf5cf4b7d875305722ec3a696(size=12.6 K), total size for store is 39.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:43:12,657 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 850d939a0dd41b63931f314702af2ad5: 2024-11-07T12:43:12,657 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., storeName=850d939a0dd41b63931f314702af2ad5/A, priority=13, startTime=1730983392152; duration=0sec 2024-11-07T12:43:12,657 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:12,657 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 850d939a0dd41b63931f314702af2ad5:A 2024-11-07T12:43:12,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/c748e44bc30a451b9e823b6b91ca4858 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/c748e44bc30a451b9e823b6b91ca4858 2024-11-07T12:43:12,666 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/c748e44bc30a451b9e823b6b91ca4858, entries=150, sequenceid=387, filesize=12.0 K 2024-11-07T12:43:12,667 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/c7f9c8fe7b194819871450c0d2bcaeef as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/c7f9c8fe7b194819871450c0d2bcaeef 2024-11-07T12:43:12,670 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/c7f9c8fe7b194819871450c0d2bcaeef, entries=150, sequenceid=387, filesize=12.0 K 2024-11-07T12:43:12,673 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=80.51 KB/82440 for 850d939a0dd41b63931f314702af2ad5 in 118ms, sequenceid=387, compaction requested=true 2024-11-07T12:43:12,673 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 850d939a0dd41b63931f314702af2ad5: 2024-11-07T12:43:12,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 850d939a0dd41b63931f314702af2ad5:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:43:12,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:43:12,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 850d939a0dd41b63931f314702af2ad5:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:43:12,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-07T12:43:12,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 850d939a0dd41b63931f314702af2ad5:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:43:12,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-07T12:43:12,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 850d939a0dd41b63931f314702af2ad5 2024-11-07T12:43:12,674 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 850d939a0dd41b63931f314702af2ad5 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-07T12:43:12,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=A 2024-11-07T12:43:12,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:12,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=B 2024-11-07T12:43:12,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:12,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=C 2024-11-07T12:43:12,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:12,676 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:43:12,676 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:43:12,679 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:43:12,680 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 850d939a0dd41b63931f314702af2ad5/C is initiating minor compaction (all files) 2024-11-07T12:43:12,680 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 850d939a0dd41b63931f314702af2ad5/C in TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:12,680 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/d4b3de077bcc472ea350170dccbf424d, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/7d7f59f69c6344b8b2334b608b9b05e1, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/c7f9c8fe7b194819871450c0d2bcaeef] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp, totalSize=36.6 K 2024-11-07T12:43:12,680 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting d4b3de077bcc472ea350170dccbf424d, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1730983391229 2024-11-07T12:43:12,680 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d7f59f69c6344b8b2334b608b9b05e1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1730983391335 2024-11-07T12:43:12,681 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting c7f9c8fe7b194819871450c0d2bcaeef, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1730983392541 2024-11-07T12:43:12,684 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/08475cf79a0c4534a1172925958abf7b is 50, key is test_row_0/A:col10/1730983392670/Put/seqid=0 2024-11-07T12:43:12,687 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39957 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:43:12,687 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): 850d939a0dd41b63931f314702af2ad5/A is initiating minor compaction (all files) 2024-11-07T12:43:12,687 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 850d939a0dd41b63931f314702af2ad5/A in TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:12,687 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/1490b50bf5cf4b7d875305722ec3a696, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/024d3bac75f44ff29b76b70530be07ac, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/2c8ffd18227e47b0a36e3282e43fb737] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp, totalSize=39.0 K 2024-11-07T12:43:12,688 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1490b50bf5cf4b7d875305722ec3a696, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1730983391229 2024-11-07T12:43:12,688 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 850d939a0dd41b63931f314702af2ad5#C#compaction#501 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:43:12,688 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 024d3bac75f44ff29b76b70530be07ac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1730983391335 2024-11-07T12:43:12,688 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c8ffd18227e47b0a36e3282e43fb737, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1730983392541 2024-11-07T12:43:12,689 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/103117f806a6451eb7e31d259e6bbcc3 is 50, key is test_row_0/C:col10/1730983392554/Put/seqid=0 2024-11-07T12:43:12,727 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 850d939a0dd41b63931f314702af2ad5#A#compaction#502 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:43:12,727 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/0980090c7fbe44cb92a405fd8da5a4f8 is 50, key is test_row_0/A:col10/1730983392554/Put/seqid=0 2024-11-07T12:43:12,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742415_1591 (size=12983) 2024-11-07T12:43:12,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742414_1590 (size=14741) 2024-11-07T12:43:12,732 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/08475cf79a0c4534a1172925958abf7b 2024-11-07T12:43:12,741 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/103117f806a6451eb7e31d259e6bbcc3 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/103117f806a6451eb7e31d259e6bbcc3 2024-11-07T12:43:12,746 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 850d939a0dd41b63931f314702af2ad5/C of 850d939a0dd41b63931f314702af2ad5 into 103117f806a6451eb7e31d259e6bbcc3(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:43:12,746 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 850d939a0dd41b63931f314702af2ad5: 2024-11-07T12:43:12,746 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., storeName=850d939a0dd41b63931f314702af2ad5/C, priority=13, startTime=1730983392674; duration=0sec 2024-11-07T12:43:12,747 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:43:12,747 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 850d939a0dd41b63931f314702af2ad5:C 2024-11-07T12:43:12,747 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:43:12,747 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:43:12,747 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 850d939a0dd41b63931f314702af2ad5/B is initiating minor compaction (all files) 2024-11-07T12:43:12,748 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 850d939a0dd41b63931f314702af2ad5/B in TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:12,748 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/dd11685f640c452e91fbc824a8fb70e7, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/22188d97e5f549d7ad395e64100f7223, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/c748e44bc30a451b9e823b6b91ca4858] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp, totalSize=36.6 K 2024-11-07T12:43:12,748 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting dd11685f640c452e91fbc824a8fb70e7, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1730983391229 2024-11-07T12:43:12,748 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 22188d97e5f549d7ad395e64100f7223, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1730983391335 2024-11-07T12:43:12,749 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting c748e44bc30a451b9e823b6b91ca4858, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1730983392541 2024-11-07T12:43:12,759 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/979e890a72a54ed59831772b0169ff8a is 50, key is test_row_0/B:col10/1730983392670/Put/seqid=0 2024-11-07T12:43:12,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742416_1592 (size=13017) 2024-11-07T12:43:12,767 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 850d939a0dd41b63931f314702af2ad5#B#compaction#504 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:43:12,767 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/fae4dece023e4e199adfd97e7181d503 is 50, key is test_row_0/B:col10/1730983392554/Put/seqid=0 2024-11-07T12:43:12,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742417_1593 (size=12301) 2024-11-07T12:43:12,783 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/979e890a72a54ed59831772b0169ff8a 2024-11-07T12:43:12,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742418_1594 (size=13017) 2024-11-07T12:43:12,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:12,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41722 deadline: 1730983452773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:12,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:12,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:12,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:12,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:12,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41752 deadline: 1730983452773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:12,788 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:12,789 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:12,789 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:12,793 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/fae4dece023e4e199adfd97e7181d503 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/fae4dece023e4e199adfd97e7181d503 2024-11-07T12:43:12,799 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 850d939a0dd41b63931f314702af2ad5/B of 850d939a0dd41b63931f314702af2ad5 into fae4dece023e4e199adfd97e7181d503(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:43:12,799 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 850d939a0dd41b63931f314702af2ad5: 2024-11-07T12:43:12,799 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., storeName=850d939a0dd41b63931f314702af2ad5/B, priority=13, startTime=1730983392674; duration=0sec 2024-11-07T12:43:12,799 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:12,799 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 850d939a0dd41b63931f314702af2ad5:B 2024-11-07T12:43:12,803 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/7f59551171924483b6da82489316136d is 50, key is test_row_0/C:col10/1730983392670/Put/seqid=0 2024-11-07T12:43:12,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742419_1595 (size=12301) 2024-11-07T12:43:12,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:12,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41722 deadline: 1730983452892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:12,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:12,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:12,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41752 deadline: 1730983452892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:12,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:12,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:12,902 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:12,902 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:12,902 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:13,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:13,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41722 deadline: 1730983453112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:13,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:13,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41752 deadline: 1730983453112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:13,124 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:13,124 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:13,124 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:13,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:13,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:13,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:13,167 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/0980090c7fbe44cb92a405fd8da5a4f8 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/0980090c7fbe44cb92a405fd8da5a4f8 2024-11-07T12:43:13,174 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 850d939a0dd41b63931f314702af2ad5/A of 850d939a0dd41b63931f314702af2ad5 into 0980090c7fbe44cb92a405fd8da5a4f8(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:43:13,174 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 850d939a0dd41b63931f314702af2ad5: 2024-11-07T12:43:13,174 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., storeName=850d939a0dd41b63931f314702af2ad5/A, priority=13, startTime=1730983392673; duration=0sec 2024-11-07T12:43:13,174 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:13,175 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 850d939a0dd41b63931f314702af2ad5:A 2024-11-07T12:43:13,209 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/7f59551171924483b6da82489316136d 2024-11-07T12:43:13,213 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/08475cf79a0c4534a1172925958abf7b as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/08475cf79a0c4534a1172925958abf7b 2024-11-07T12:43:13,216 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/08475cf79a0c4534a1172925958abf7b, entries=200, sequenceid=405, filesize=14.4 K 2024-11-07T12:43:13,217 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/979e890a72a54ed59831772b0169ff8a as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/979e890a72a54ed59831772b0169ff8a 2024-11-07T12:43:13,221 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/979e890a72a54ed59831772b0169ff8a, entries=150, sequenceid=405, filesize=12.0 K 2024-11-07T12:43:13,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-07T12:43:13,222 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/7f59551171924483b6da82489316136d as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/7f59551171924483b6da82489316136d 2024-11-07T12:43:13,222 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:43:13,223 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:43:13,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-07T12:43:13,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-07T12:43:13,226 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:43:13,227 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:43:13,227 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:43:13,232 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/7f59551171924483b6da82489316136d, entries=150, sequenceid=405, filesize=12.0 K 2024-11-07T12:43:13,233 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 850d939a0dd41b63931f314702af2ad5 in 559ms, sequenceid=405, compaction requested=false 2024-11-07T12:43:13,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 850d939a0dd41b63931f314702af2ad5: 2024-11-07T12:43:13,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-07T12:43:13,380 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-07T12:43:13,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:13,380 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing 850d939a0dd41b63931f314702af2ad5 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-07T12:43:13,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=A 2024-11-07T12:43:13,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:13,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=B 2024-11-07T12:43:13,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:13,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=C 2024-11-07T12:43:13,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:13,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/2e273050c85a4afea0092a8a8af9f037 is 50, key is test_row_0/A:col10/1730983392761/Put/seqid=0 2024-11-07T12:43:13,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 850d939a0dd41b63931f314702af2ad5 2024-11-07T12:43:13,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. as already flushing 2024-11-07T12:43:13,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742420_1596 (size=12301) 2024-11-07T12:43:13,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:13,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41722 deadline: 1730983453529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:13,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:13,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:13,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:13,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-07T12:43:13,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:13,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41752 deadline: 1730983453539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:13,554 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:13,554 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:13,554 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:13,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:13,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41722 deadline: 1730983453652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:13,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:13,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:13,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:13,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:13,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41752 deadline: 1730983453662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:13,673 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:13,673 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:13,673 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:13,849 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/2e273050c85a4afea0092a8a8af9f037 2024-11-07T12:43:13,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-07T12:43:13,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/ed696a6ee45741f3881ea308257b0416 is 50, key is test_row_0/B:col10/1730983392761/Put/seqid=0 2024-11-07T12:43:13,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742421_1597 (size=12301) 2024-11-07T12:43:13,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:13,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41722 deadline: 1730983453871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:13,888 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:13,888 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:13,888 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:13,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:13,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41752 deadline: 1730983453882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:13,894 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:13,894 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:13,895 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:14,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:14,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41722 deadline: 1730983454192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:14,202 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:14,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:14,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:14,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41752 deadline: 1730983454202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:14,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:14,202 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:14,202 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:14,202 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:14,269 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/ed696a6ee45741f3881ea308257b0416 2024-11-07T12:43:14,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/7667a5ba169c4b92bccbe628870dbab1 is 50, key is test_row_0/C:col10/1730983392761/Put/seqid=0 2024-11-07T12:43:14,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742422_1598 (size=12301) 2024-11-07T12:43:14,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-07T12:43:14,713 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/7667a5ba169c4b92bccbe628870dbab1 2024-11-07T12:43:14,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:14,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41722 deadline: 1730983454712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:14,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:14,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:14,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:14,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41752 deadline: 1730983454712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:14,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:14,726 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:14,726 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:14,726 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:14,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/2e273050c85a4afea0092a8a8af9f037 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/2e273050c85a4afea0092a8a8af9f037 2024-11-07T12:43:14,730 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/2e273050c85a4afea0092a8a8af9f037, entries=150, sequenceid=423, filesize=12.0 K 2024-11-07T12:43:14,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/ed696a6ee45741f3881ea308257b0416 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/ed696a6ee45741f3881ea308257b0416 2024-11-07T12:43:14,735 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/ed696a6ee45741f3881ea308257b0416, entries=150, sequenceid=423, filesize=12.0 K 2024-11-07T12:43:14,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/7667a5ba169c4b92bccbe628870dbab1 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/7667a5ba169c4b92bccbe628870dbab1 2024-11-07T12:43:14,740 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/7667a5ba169c4b92bccbe628870dbab1, entries=150, sequenceid=423, filesize=12.0 K 2024-11-07T12:43:14,741 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 850d939a0dd41b63931f314702af2ad5 in 1361ms, sequenceid=423, compaction requested=true 2024-11-07T12:43:14,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for 850d939a0dd41b63931f314702af2ad5: 2024-11-07T12:43:14,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:14,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-07T12:43:14,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-07T12:43:14,745 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-07T12:43:14,745 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5160 sec 2024-11-07T12:43:14,747 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 1.5220 sec 2024-11-07T12:43:15,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-07T12:43:15,371 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:43:15,372 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:43:15,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees 2024-11-07T12:43:15,378 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:43:15,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-07T12:43:15,379 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:43:15,379 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:43:15,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-07T12:43:15,532 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-07T12:43:15,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:15,532 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2902): Flushing 850d939a0dd41b63931f314702af2ad5 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-07T12:43:15,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=A 2024-11-07T12:43:15,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:15,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=B 2024-11-07T12:43:15,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:15,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=C 2024-11-07T12:43:15,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:15,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/f9069431666341b58ebf6b6d5dc247cd is 50, key is test_row_0/A:col10/1730983393527/Put/seqid=0 2024-11-07T12:43:15,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742423_1599 (size=12301) 2024-11-07T12:43:15,613 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/f9069431666341b58ebf6b6d5dc247cd 2024-11-07T12:43:15,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/a6c0d7070b544118bb72c89da7a10472 is 50, key is test_row_0/B:col10/1730983393527/Put/seqid=0 2024-11-07T12:43:15,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-07T12:43:15,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742424_1600 (size=12301) 2024-11-07T12:43:15,719 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/a6c0d7070b544118bb72c89da7a10472 2024-11-07T12:43:15,738 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. as already flushing 2024-11-07T12:43:15,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/a21c8917ccb64939bf3c5676ee522fe4 is 50, key is test_row_0/C:col10/1730983393527/Put/seqid=0 2024-11-07T12:43:15,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 850d939a0dd41b63931f314702af2ad5 2024-11-07T12:43:15,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742425_1601 (size=12301) 2024-11-07T12:43:15,757 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/a21c8917ccb64939bf3c5676ee522fe4 2024-11-07T12:43:15,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/f9069431666341b58ebf6b6d5dc247cd as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/f9069431666341b58ebf6b6d5dc247cd 2024-11-07T12:43:15,766 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/f9069431666341b58ebf6b6d5dc247cd, entries=150, sequenceid=440, filesize=12.0 K 2024-11-07T12:43:15,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/a6c0d7070b544118bb72c89da7a10472 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/a6c0d7070b544118bb72c89da7a10472 2024-11-07T12:43:15,801 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/a6c0d7070b544118bb72c89da7a10472, entries=150, sequenceid=440, filesize=12.0 K 2024-11-07T12:43:15,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/a21c8917ccb64939bf3c5676ee522fe4 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/a21c8917ccb64939bf3c5676ee522fe4 2024-11-07T12:43:15,829 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/a21c8917ccb64939bf3c5676ee522fe4, entries=150, sequenceid=440, filesize=12.0 K 2024-11-07T12:43:15,831 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=67.09 KB/68700 for 850d939a0dd41b63931f314702af2ad5 in 299ms, sequenceid=440, compaction requested=true 2024-11-07T12:43:15,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2603): Flush status journal for 850d939a0dd41b63931f314702af2ad5: 2024-11-07T12:43:15,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:15,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-11-07T12:43:15,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-11-07T12:43:15,844 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-07T12:43:15,844 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 463 msec 2024-11-07T12:43:15,846 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees in 473 msec 2024-11-07T12:43:15,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 850d939a0dd41b63931f314702af2ad5 2024-11-07T12:43:15,847 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 850d939a0dd41b63931f314702af2ad5 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-07T12:43:15,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=A 2024-11-07T12:43:15,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:15,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=B 2024-11-07T12:43:15,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:15,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=C 2024-11-07T12:43:15,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:15,863 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/11ff98c2a23a4dd4a4e266816400f285 is 50, key is test_row_0/A:col10/1730983395836/Put/seqid=0 2024-11-07T12:43:15,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742426_1602 (size=14741) 2024-11-07T12:43:15,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:15,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41694 deadline: 1730983455955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:15,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:15,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:15,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:15,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:15,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41730 deadline: 1730983455962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:15,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:15,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1730983455967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:15,974 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to db9ad1cb6cf9,34293,1730983260217 for 'test_row_0' in 850d939a0dd41b63931f314702af2ad5 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8269 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:43:15,974 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:15,974 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:15,975 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:15,975 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:15,975 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:15,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:15,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:15,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41752 deadline: 1730983455968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:15,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:15,990 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:15,990 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:15,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41722 deadline: 1730983455969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:15,990 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:15,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:15,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:15,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:16,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-07T12:43:16,001 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:43:16,002 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:43:16,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees 2024-11-07T12:43:16,004 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:43:16,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-07T12:43:16,005 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:43:16,006 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:43:16,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:16,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1730983456082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:16,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:16,083 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:16,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41694 deadline: 1730983456082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:16,083 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:16,083 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:16,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:16,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:16,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:16,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-07T12:43:16,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:16,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41722 deadline: 1730983456102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:16,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:16,116 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:16,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41752 deadline: 1730983456102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:16,116 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:16,116 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:16,117 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:16,117 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:16,117 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:16,158 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-07T12:43:16,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:16,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. as already flushing 2024-11-07T12:43:16,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:16,158 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:16,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:16,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:16,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:16,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1730983456292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:16,306 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:16,306 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:16,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:16,306 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:16,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41694 deadline: 1730983456292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:16,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:16,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:16,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:16,314 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-07T12:43:16,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:16,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. as already flushing 2024-11-07T12:43:16,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:16,314 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:16,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:16,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:16,326 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=454 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/11ff98c2a23a4dd4a4e266816400f285 2024-11-07T12:43:16,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-07T12:43:16,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:16,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41722 deadline: 1730983456326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:16,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:16,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41752 deadline: 1730983456326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:16,336 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:16,336 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:16,336 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:16,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:16,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:16,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:16,367 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/9a4df53911d242128baf4f7472a5653f is 50, key is test_row_0/B:col10/1730983395836/Put/seqid=0 2024-11-07T12:43:16,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742427_1603 (size=12301) 2024-11-07T12:43:16,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-07T12:43:16,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:16,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. as already flushing 2024-11-07T12:43:16,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:16,472 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:16,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:16,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:16,626 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-07T12:43:16,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:16,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. as already flushing 2024-11-07T12:43:16,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:16,627 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:16,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:16,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:16,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:16,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1730983456614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:16,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:16,631 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:16,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41694 deadline: 1730983456614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:16,631 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:16,631 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:16,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:16,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:16,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:16,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-07T12:43:16,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:16,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41752 deadline: 1730983456641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:16,653 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:16,653 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:16,653 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:16,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:16,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41722 deadline: 1730983456653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:16,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:16,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:16,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:16,782 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=454 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/9a4df53911d242128baf4f7472a5653f 2024-11-07T12:43:16,783 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-07T12:43:16,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:16,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. as already flushing 2024-11-07T12:43:16,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:16,784 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:16,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:16,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:16,800 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/f159b9f8ab6b488f92d41bfb6cff228a is 50, key is test_row_0/C:col10/1730983395836/Put/seqid=0 2024-11-07T12:43:16,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742428_1604 (size=12301) 2024-11-07T12:43:16,803 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=454 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/f159b9f8ab6b488f92d41bfb6cff228a 2024-11-07T12:43:16,807 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/11ff98c2a23a4dd4a4e266816400f285 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/11ff98c2a23a4dd4a4e266816400f285 2024-11-07T12:43:16,818 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/11ff98c2a23a4dd4a4e266816400f285, entries=200, sequenceid=454, filesize=14.4 K 2024-11-07T12:43:16,822 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/9a4df53911d242128baf4f7472a5653f as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/9a4df53911d242128baf4f7472a5653f 2024-11-07T12:43:16,826 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/9a4df53911d242128baf4f7472a5653f, entries=150, sequenceid=454, filesize=12.0 K 2024-11-07T12:43:16,826 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/f159b9f8ab6b488f92d41bfb6cff228a as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/f159b9f8ab6b488f92d41bfb6cff228a 2024-11-07T12:43:16,835 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/f159b9f8ab6b488f92d41bfb6cff228a, entries=150, sequenceid=454, filesize=12.0 K 2024-11-07T12:43:16,836 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 850d939a0dd41b63931f314702af2ad5 in 989ms, sequenceid=454, compaction requested=true 2024-11-07T12:43:16,836 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 850d939a0dd41b63931f314702af2ad5: 2024-11-07T12:43:16,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 850d939a0dd41b63931f314702af2ad5:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:43:16,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:16,836 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-07T12:43:16,836 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-07T12:43:16,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 850d939a0dd41b63931f314702af2ad5:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:43:16,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:16,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 850d939a0dd41b63931f314702af2ad5:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:43:16,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:43:16,838 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 67101 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-07T12:43:16,838 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): 850d939a0dd41b63931f314702af2ad5/A is initiating minor compaction (all files) 2024-11-07T12:43:16,838 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 850d939a0dd41b63931f314702af2ad5/A in TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:16,838 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/0980090c7fbe44cb92a405fd8da5a4f8, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/08475cf79a0c4534a1172925958abf7b, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/2e273050c85a4afea0092a8a8af9f037, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/f9069431666341b58ebf6b6d5dc247cd, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/11ff98c2a23a4dd4a4e266816400f285] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp, totalSize=65.5 K 2024-11-07T12:43:16,838 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62221 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-07T12:43:16,838 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 850d939a0dd41b63931f314702af2ad5/B is initiating minor compaction (all files) 2024-11-07T12:43:16,838 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0980090c7fbe44cb92a405fd8da5a4f8, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1730983392541 2024-11-07T12:43:16,838 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 850d939a0dd41b63931f314702af2ad5/B in TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:16,838 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/fae4dece023e4e199adfd97e7181d503, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/979e890a72a54ed59831772b0169ff8a, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/ed696a6ee45741f3881ea308257b0416, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/a6c0d7070b544118bb72c89da7a10472, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/9a4df53911d242128baf4f7472a5653f] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp, totalSize=60.8 K 2024-11-07T12:43:16,839 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 08475cf79a0c4534a1172925958abf7b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1730983392635 2024-11-07T12:43:16,839 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting fae4dece023e4e199adfd97e7181d503, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1730983392541 2024-11-07T12:43:16,839 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e273050c85a4afea0092a8a8af9f037, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1730983392754 2024-11-07T12:43:16,842 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 979e890a72a54ed59831772b0169ff8a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1730983392635 2024-11-07T12:43:16,843 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting ed696a6ee45741f3881ea308257b0416, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1730983392754 2024-11-07T12:43:16,843 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting f9069431666341b58ebf6b6d5dc247cd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=440, earliestPutTs=1730983393483 2024-11-07T12:43:16,843 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting a6c0d7070b544118bb72c89da7a10472, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=440, earliestPutTs=1730983393483 2024-11-07T12:43:16,843 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11ff98c2a23a4dd4a4e266816400f285, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=454, earliestPutTs=1730983395799 2024-11-07T12:43:16,844 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a4df53911d242128baf4f7472a5653f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=454, earliestPutTs=1730983395799 2024-11-07T12:43:16,860 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 850d939a0dd41b63931f314702af2ad5#A#compaction#515 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:43:16,861 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/a6278f4f4c374a21bbca6e2e0d8166fd is 50, key is test_row_0/A:col10/1730983395836/Put/seqid=0 2024-11-07T12:43:16,868 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 850d939a0dd41b63931f314702af2ad5#B#compaction#516 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:43:16,868 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/45a88ba256374cd398ca0362df121c13 is 50, key is test_row_0/B:col10/1730983395836/Put/seqid=0 2024-11-07T12:43:16,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742429_1605 (size=13187) 2024-11-07T12:43:16,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742430_1606 (size=13187) 2024-11-07T12:43:16,942 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-07T12:43:16,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:16,942 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2902): Flushing 850d939a0dd41b63931f314702af2ad5 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-07T12:43:16,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=A 2024-11-07T12:43:16,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:16,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=B 2024-11-07T12:43:16,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:16,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=C 2024-11-07T12:43:16,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:16,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/a90a50348a8344549c1584f2417415a5 is 50, key is test_row_0/A:col10/1730983395950/Put/seqid=0 2024-11-07T12:43:16,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742431_1607 (size=12301) 2024-11-07T12:43:16,969 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=472 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/a90a50348a8344549c1584f2417415a5 2024-11-07T12:43:16,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/bba81c33d709419f8318529199ab851e is 50, key is test_row_0/B:col10/1730983395950/Put/seqid=0 2024-11-07T12:43:17,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742432_1608 (size=12301) 2024-11-07T12:43:17,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-07T12:43:17,147 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. as already flushing 2024-11-07T12:43:17,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 850d939a0dd41b63931f314702af2ad5 2024-11-07T12:43:17,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:17,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1730983457206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:17,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:17,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41722 deadline: 1730983457206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:17,231 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:17,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:17,231 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:17,231 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:17,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41752 deadline: 1730983457209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:17,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:17,231 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:17,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41694 deadline: 1730983457209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:17,231 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:17,231 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:17,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:17,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:17,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:17,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:17,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:17,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:17,288 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/a6278f4f4c374a21bbca6e2e0d8166fd as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/a6278f4f4c374a21bbca6e2e0d8166fd 2024-11-07T12:43:17,291 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/45a88ba256374cd398ca0362df121c13 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/45a88ba256374cd398ca0362df121c13 2024-11-07T12:43:17,295 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 850d939a0dd41b63931f314702af2ad5/A of 850d939a0dd41b63931f314702af2ad5 into a6278f4f4c374a21bbca6e2e0d8166fd(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:43:17,295 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 850d939a0dd41b63931f314702af2ad5: 2024-11-07T12:43:17,295 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., storeName=850d939a0dd41b63931f314702af2ad5/A, priority=11, startTime=1730983396836; duration=0sec 2024-11-07T12:43:17,295 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:43:17,296 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 850d939a0dd41b63931f314702af2ad5:A 2024-11-07T12:43:17,296 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-07T12:43:17,299 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 850d939a0dd41b63931f314702af2ad5/B of 850d939a0dd41b63931f314702af2ad5 into 45a88ba256374cd398ca0362df121c13(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:43:17,299 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 850d939a0dd41b63931f314702af2ad5: 2024-11-07T12:43:17,299 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., storeName=850d939a0dd41b63931f314702af2ad5/B, priority=11, startTime=1730983396836; duration=0sec 2024-11-07T12:43:17,299 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:17,299 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 850d939a0dd41b63931f314702af2ad5:B 2024-11-07T12:43:17,300 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62187 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-07T12:43:17,301 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): 850d939a0dd41b63931f314702af2ad5/C is initiating minor compaction (all files) 2024-11-07T12:43:17,301 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 850d939a0dd41b63931f314702af2ad5/C in TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:17,301 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/103117f806a6451eb7e31d259e6bbcc3, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/7f59551171924483b6da82489316136d, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/7667a5ba169c4b92bccbe628870dbab1, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/a21c8917ccb64939bf3c5676ee522fe4, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/f159b9f8ab6b488f92d41bfb6cff228a] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp, totalSize=60.7 K 2024-11-07T12:43:17,303 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 103117f806a6451eb7e31d259e6bbcc3, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1730983392541 2024-11-07T12:43:17,304 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f59551171924483b6da82489316136d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1730983392635 2024-11-07T12:43:17,304 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7667a5ba169c4b92bccbe628870dbab1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1730983392754 2024-11-07T12:43:17,304 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting a21c8917ccb64939bf3c5676ee522fe4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=440, earliestPutTs=1730983393483 2024-11-07T12:43:17,304 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting f159b9f8ab6b488f92d41bfb6cff228a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=454, earliestPutTs=1730983395799 2024-11-07T12:43:17,319 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 850d939a0dd41b63931f314702af2ad5#C#compaction#519 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:43:17,319 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/282c586e976e4fe2ab919dad59c99ec6 is 50, key is test_row_0/C:col10/1730983395836/Put/seqid=0 2024-11-07T12:43:17,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742433_1609 (size=13153) 2024-11-07T12:43:17,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:17,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1730983457341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:17,347 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:17,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:17,347 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:17,347 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:17,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41752 deadline: 1730983457342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:17,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:17,347 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:17,347 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:17,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41722 deadline: 1730983457342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:17,347 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:17,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:17,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:17,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:17,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:17,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41694 deadline: 1730983457344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:17,350 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:17,350 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:17,351 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:17,429 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=472 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/bba81c33d709419f8318529199ab851e 2024-11-07T12:43:17,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/7ed61756dbc04853b62df506914a6be9 is 50, key is test_row_0/C:col10/1730983395950/Put/seqid=0 2024-11-07T12:43:17,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742434_1610 (size=12301) 2024-11-07T12:43:17,473 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=472 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/7ed61756dbc04853b62df506914a6be9 2024-11-07T12:43:17,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/a90a50348a8344549c1584f2417415a5 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/a90a50348a8344549c1584f2417415a5 2024-11-07T12:43:17,490 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/a90a50348a8344549c1584f2417415a5, entries=150, sequenceid=472, filesize=12.0 K 2024-11-07T12:43:17,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/bba81c33d709419f8318529199ab851e as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/bba81c33d709419f8318529199ab851e 2024-11-07T12:43:17,494 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/bba81c33d709419f8318529199ab851e, entries=150, sequenceid=472, filesize=12.0 K 2024-11-07T12:43:17,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/7ed61756dbc04853b62df506914a6be9 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/7ed61756dbc04853b62df506914a6be9 2024-11-07T12:43:17,517 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/7ed61756dbc04853b62df506914a6be9, entries=150, sequenceid=472, filesize=12.0 K 2024-11-07T12:43:17,518 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 850d939a0dd41b63931f314702af2ad5 in 576ms, sequenceid=472, compaction requested=false 2024-11-07T12:43:17,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2603): Flush status journal for 850d939a0dd41b63931f314702af2ad5: 2024-11-07T12:43:17,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:17,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=140 2024-11-07T12:43:17,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4169): Remote procedure done, pid=140 2024-11-07T12:43:17,524 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-07T12:43:17,524 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5160 sec 2024-11-07T12:43:17,526 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees in 1.5230 sec 2024-11-07T12:43:17,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 850d939a0dd41b63931f314702af2ad5 2024-11-07T12:43:17,558 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 850d939a0dd41b63931f314702af2ad5 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-07T12:43:17,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=A 2024-11-07T12:43:17,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:17,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=B 2024-11-07T12:43:17,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:17,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=C 2024-11-07T12:43:17,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:17,582 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/56c44e94c497468b9dfa8b49e444218d is 50, key is test_row_0/A:col10/1730983397206/Put/seqid=0 2024-11-07T12:43:17,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742435_1611 (size=14741) 2024-11-07T12:43:17,591 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=489 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/56c44e94c497468b9dfa8b49e444218d 2024-11-07T12:43:17,601 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/ba592b7fbd5e4a1fbaea0a8d2d336840 is 50, key is test_row_0/B:col10/1730983397206/Put/seqid=0 2024-11-07T12:43:17,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742436_1612 (size=12301) 2024-11-07T12:43:17,628 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=489 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/ba592b7fbd5e4a1fbaea0a8d2d336840 2024-11-07T12:43:17,635 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/09d14d162e134ac0990b1d8c364ddf19 is 50, key is test_row_0/C:col10/1730983397206/Put/seqid=0 2024-11-07T12:43:17,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:17,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41752 deadline: 1730983457625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:17,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:17,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41694 deadline: 1730983457625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:17,647 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:17,647 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:17,647 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:17,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:17,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1730983457627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:17,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:17,648 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:17,648 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:17,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41722 deadline: 1730983457630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:17,648 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:17,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:17,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:17,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:17,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:17,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:17,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:17,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742437_1613 (size=12301) 2024-11-07T12:43:17,650 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=489 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/09d14d162e134ac0990b1d8c364ddf19 2024-11-07T12:43:17,653 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/56c44e94c497468b9dfa8b49e444218d as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/56c44e94c497468b9dfa8b49e444218d 2024-11-07T12:43:17,659 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/56c44e94c497468b9dfa8b49e444218d, entries=200, sequenceid=489, filesize=14.4 K 2024-11-07T12:43:17,659 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/ba592b7fbd5e4a1fbaea0a8d2d336840 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/ba592b7fbd5e4a1fbaea0a8d2d336840 2024-11-07T12:43:17,663 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/ba592b7fbd5e4a1fbaea0a8d2d336840, entries=150, sequenceid=489, filesize=12.0 K 2024-11-07T12:43:17,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/09d14d162e134ac0990b1d8c364ddf19 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/09d14d162e134ac0990b1d8c364ddf19 2024-11-07T12:43:17,669 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/09d14d162e134ac0990b1d8c364ddf19, entries=150, sequenceid=489, filesize=12.0 K 2024-11-07T12:43:17,671 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 850d939a0dd41b63931f314702af2ad5 in 114ms, sequenceid=489, compaction requested=true 2024-11-07T12:43:17,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 850d939a0dd41b63931f314702af2ad5: 2024-11-07T12:43:17,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 850d939a0dd41b63931f314702af2ad5:A, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:43:17,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:17,671 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:43:17,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 850d939a0dd41b63931f314702af2ad5:B, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:43:17,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:43:17,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 850d939a0dd41b63931f314702af2ad5:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:43:17,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-07T12:43:17,673 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40229 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:43:17,673 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 850d939a0dd41b63931f314702af2ad5/A is initiating minor compaction (all files) 2024-11-07T12:43:17,673 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 850d939a0dd41b63931f314702af2ad5/A in TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:17,673 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/a6278f4f4c374a21bbca6e2e0d8166fd, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/a90a50348a8344549c1584f2417415a5, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/56c44e94c497468b9dfa8b49e444218d] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp, totalSize=39.3 K 2024-11-07T12:43:17,673 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting a6278f4f4c374a21bbca6e2e0d8166fd, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=454, earliestPutTs=1730983395799 2024-11-07T12:43:17,674 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting a90a50348a8344549c1584f2417415a5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=472, earliestPutTs=1730983395950 2024-11-07T12:43:17,674 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 56c44e94c497468b9dfa8b49e444218d, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=489, earliestPutTs=1730983397147 2024-11-07T12:43:17,689 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 850d939a0dd41b63931f314702af2ad5#A#compaction#524 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:43:17,690 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/f6ecd18bef2b40eab22598b4730223cd is 50, key is test_row_0/A:col10/1730983397206/Put/seqid=0 2024-11-07T12:43:17,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742438_1614 (size=13289) 2024-11-07T12:43:17,705 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/f6ecd18bef2b40eab22598b4730223cd as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/f6ecd18bef2b40eab22598b4730223cd 2024-11-07T12:43:17,713 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 850d939a0dd41b63931f314702af2ad5/A of 850d939a0dd41b63931f314702af2ad5 into f6ecd18bef2b40eab22598b4730223cd(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:43:17,713 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 850d939a0dd41b63931f314702af2ad5: 2024-11-07T12:43:17,713 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., storeName=850d939a0dd41b63931f314702af2ad5/A, priority=13, startTime=1730983397671; duration=0sec 2024-11-07T12:43:17,713 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-07T12:43:17,713 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 850d939a0dd41b63931f314702af2ad5:A 2024-11-07T12:43:17,713 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 7 store files, 5 compacting, 2 eligible, 16 blocking 2024-11-07T12:43:17,714 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-07T12:43:17,714 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-07T12:43:17,714 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. because compaction request was cancelled 2024-11-07T12:43:17,714 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 850d939a0dd41b63931f314702af2ad5:C 2024-11-07T12:43:17,714 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:43:17,715 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:43:17,715 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 850d939a0dd41b63931f314702af2ad5/B is initiating minor compaction (all files) 2024-11-07T12:43:17,715 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 850d939a0dd41b63931f314702af2ad5/B in TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:17,715 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/45a88ba256374cd398ca0362df121c13, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/bba81c33d709419f8318529199ab851e, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/ba592b7fbd5e4a1fbaea0a8d2d336840] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp, totalSize=36.9 K 2024-11-07T12:43:17,715 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 45a88ba256374cd398ca0362df121c13, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=454, earliestPutTs=1730983395799 2024-11-07T12:43:17,716 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting bba81c33d709419f8318529199ab851e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=472, earliestPutTs=1730983395950 2024-11-07T12:43:17,716 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting ba592b7fbd5e4a1fbaea0a8d2d336840, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=489, earliestPutTs=1730983397147 2024-11-07T12:43:17,723 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 850d939a0dd41b63931f314702af2ad5#B#compaction#525 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:43:17,724 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/c6673fc2cb4949459802fba9bb0a18fc is 50, key is test_row_0/B:col10/1730983397206/Put/seqid=0 2024-11-07T12:43:17,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742439_1615 (size=13289) 2024-11-07T12:43:17,739 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/282c586e976e4fe2ab919dad59c99ec6 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/282c586e976e4fe2ab919dad59c99ec6 2024-11-07T12:43:17,750 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 850d939a0dd41b63931f314702af2ad5/C of 850d939a0dd41b63931f314702af2ad5 into 282c586e976e4fe2ab919dad59c99ec6(size=12.8 K), total size for store is 36.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:43:17,750 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 850d939a0dd41b63931f314702af2ad5: 2024-11-07T12:43:17,750 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., storeName=850d939a0dd41b63931f314702af2ad5/C, priority=11, startTime=1730983396837; duration=0sec 2024-11-07T12:43:17,751 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:17,751 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 850d939a0dd41b63931f314702af2ad5:C 2024-11-07T12:43:17,753 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/c6673fc2cb4949459802fba9bb0a18fc as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/c6673fc2cb4949459802fba9bb0a18fc 2024-11-07T12:43:17,757 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 850d939a0dd41b63931f314702af2ad5/B of 850d939a0dd41b63931f314702af2ad5 into c6673fc2cb4949459802fba9bb0a18fc(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:43:17,757 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 850d939a0dd41b63931f314702af2ad5: 2024-11-07T12:43:17,757 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., storeName=850d939a0dd41b63931f314702af2ad5/B, priority=13, startTime=1730983397671; duration=0sec 2024-11-07T12:43:17,757 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:17,757 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 850d939a0dd41b63931f314702af2ad5:B 2024-11-07T12:43:17,757 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 850d939a0dd41b63931f314702af2ad5 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-07T12:43:17,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=A 2024-11-07T12:43:17,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:17,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=B 2024-11-07T12:43:17,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:17,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 850d939a0dd41b63931f314702af2ad5 2024-11-07T12:43:17,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=C 2024-11-07T12:43:17,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:17,767 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/0135c0b0b316427c890f239ac86d2d30 is 50, key is test_row_0/A:col10/1730983397755/Put/seqid=0 2024-11-07T12:43:17,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742440_1616 (size=14741) 2024-11-07T12:43:17,782 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=511 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/0135c0b0b316427c890f239ac86d2d30 2024-11-07T12:43:17,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:17,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41694 deadline: 1730983457778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:17,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:17,784 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:17,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41722 deadline: 1730983457779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:17,784 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:17,784 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:17,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:17,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:17,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:17,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:17,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1730983457783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:17,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:17,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41752 deadline: 1730983457785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:17,800 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:17,800 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:17,800 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:17,800 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:17,801 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:17,801 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:17,810 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/f048965776a14629a3f0364ffb697c9b is 50, key is test_row_0/B:col10/1730983397755/Put/seqid=0 2024-11-07T12:43:17,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742441_1617 (size=12301) 2024-11-07T12:43:17,818 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=511 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/f048965776a14629a3f0364ffb697c9b 2024-11-07T12:43:17,830 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/b0310ca93d1f42fe9f4283299a76ffba is 50, key is test_row_0/C:col10/1730983397755/Put/seqid=0 2024-11-07T12:43:17,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742442_1618 (size=12301) 2024-11-07T12:43:17,840 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=511 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/b0310ca93d1f42fe9f4283299a76ffba 2024-11-07T12:43:17,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/0135c0b0b316427c890f239ac86d2d30 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/0135c0b0b316427c890f239ac86d2d30 2024-11-07T12:43:17,856 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/0135c0b0b316427c890f239ac86d2d30, entries=200, sequenceid=511, filesize=14.4 K 2024-11-07T12:43:17,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/f048965776a14629a3f0364ffb697c9b as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/f048965776a14629a3f0364ffb697c9b 2024-11-07T12:43:17,867 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/f048965776a14629a3f0364ffb697c9b, entries=150, sequenceid=511, filesize=12.0 K 2024-11-07T12:43:17,869 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/b0310ca93d1f42fe9f4283299a76ffba as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/b0310ca93d1f42fe9f4283299a76ffba 2024-11-07T12:43:17,873 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/b0310ca93d1f42fe9f4283299a76ffba, entries=150, sequenceid=511, filesize=12.0 K 2024-11-07T12:43:17,878 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 850d939a0dd41b63931f314702af2ad5 in 121ms, sequenceid=511, compaction requested=true 2024-11-07T12:43:17,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 850d939a0dd41b63931f314702af2ad5: 2024-11-07T12:43:17,878 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-07T12:43:17,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 850d939a0dd41b63931f314702af2ad5:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:43:17,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:17,879 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-07T12:43:17,879 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-07T12:43:17,879 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-07T12:43:17,879 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. because compaction request was cancelled 2024-11-07T12:43:17,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 850d939a0dd41b63931f314702af2ad5:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:43:17,879 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 850d939a0dd41b63931f314702af2ad5:A 2024-11-07T12:43:17,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:17,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 850d939a0dd41b63931f314702af2ad5:C, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:43:17,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:43:17,879 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:43:17,879 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-07T12:43:17,879 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-07T12:43:17,879 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. because compaction request was cancelled 2024-11-07T12:43:17,879 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 850d939a0dd41b63931f314702af2ad5:B 2024-11-07T12:43:17,885 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50056 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:43:17,885 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): 850d939a0dd41b63931f314702af2ad5/C is initiating minor compaction (all files) 2024-11-07T12:43:17,885 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 850d939a0dd41b63931f314702af2ad5/C in TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:17,886 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/282c586e976e4fe2ab919dad59c99ec6, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/7ed61756dbc04853b62df506914a6be9, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/09d14d162e134ac0990b1d8c364ddf19, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/b0310ca93d1f42fe9f4283299a76ffba] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp, totalSize=48.9 K 2024-11-07T12:43:17,886 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 282c586e976e4fe2ab919dad59c99ec6, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=454, earliestPutTs=1730983395799 2024-11-07T12:43:17,886 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ed61756dbc04853b62df506914a6be9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=472, earliestPutTs=1730983395950 2024-11-07T12:43:17,887 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09d14d162e134ac0990b1d8c364ddf19, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=489, earliestPutTs=1730983397147 2024-11-07T12:43:17,890 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0310ca93d1f42fe9f4283299a76ffba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=511, earliestPutTs=1730983397619 2024-11-07T12:43:17,903 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 850d939a0dd41b63931f314702af2ad5 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-07T12:43:17,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 850d939a0dd41b63931f314702af2ad5 2024-11-07T12:43:17,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=A 2024-11-07T12:43:17,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:17,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=B 2024-11-07T12:43:17,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:17,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=C 2024-11-07T12:43:17,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:17,915 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 850d939a0dd41b63931f314702af2ad5#C#compaction#529 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:43:17,915 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/0626691b740e49b69a54da0b8fe7eeb7 is 50, key is test_row_0/C:col10/1730983397755/Put/seqid=0 2024-11-07T12:43:17,926 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/913d394be1b24fe5afe6a35eb45eaff4 is 50, key is test_row_0/A:col10/1730983397901/Put/seqid=0 2024-11-07T12:43:17,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742443_1619 (size=13289) 2024-11-07T12:43:17,946 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/0626691b740e49b69a54da0b8fe7eeb7 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/0626691b740e49b69a54da0b8fe7eeb7 2024-11-07T12:43:17,958 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 850d939a0dd41b63931f314702af2ad5/C of 850d939a0dd41b63931f314702af2ad5 into 0626691b740e49b69a54da0b8fe7eeb7(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:43:17,958 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 850d939a0dd41b63931f314702af2ad5: 2024-11-07T12:43:17,958 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., storeName=850d939a0dd41b63931f314702af2ad5/C, priority=12, startTime=1730983397879; duration=0sec 2024-11-07T12:43:17,958 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:17,958 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 850d939a0dd41b63931f314702af2ad5:C 2024-11-07T12:43:17,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742444_1620 (size=14741) 2024-11-07T12:43:17,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:17,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41722 deadline: 1730983457959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:17,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:17,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41752 deadline: 1730983457961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:17,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:17,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41694 deadline: 1730983457964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:17,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:17,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1730983457969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:17,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:17,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:17,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:17,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:17,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:17,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:17,982 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:17,982 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:17,982 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:17,982 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:17,982 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:17,982 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:18,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:18,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41694 deadline: 1730983458092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:18,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:18,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1730983458092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:18,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:18,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:18,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:18,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:18,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41752 deadline: 1730983458092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:18,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:18,108 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:18,108 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:18,108 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:18,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41722 deadline: 1730983458093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:18,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:18,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:18,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:18,110 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:18,110 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:18,110 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:18,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-07T12:43:18,158 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:43:18,161 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:43:18,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees 2024-11-07T12:43:18,163 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=141, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:43:18,164 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=141, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:43:18,164 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:43:18,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-07T12:43:18,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-07T12:43:18,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-07T12:43:18,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:18,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. as already flushing 2024-11-07T12:43:18,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:18,318 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:18,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:18,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:18,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:18,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41694 deadline: 1730983458312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:18,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:18,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41722 deadline: 1730983458312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:18,323 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:18,323 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:18,324 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:18,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:18,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:18,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:18,324 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:18,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41752 deadline: 1730983458313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:18,324 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:18,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1730983458321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:18,325 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:18,325 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:18,325 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:18,326 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:18,326 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:18,326 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:18,359 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=526 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/913d394be1b24fe5afe6a35eb45eaff4 2024-11-07T12:43:18,383 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/2c2b1f5daf6447809e7fae99d6be1edd is 50, key is test_row_0/B:col10/1730983397901/Put/seqid=0 2024-11-07T12:43:18,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742445_1621 (size=12301) 2024-11-07T12:43:18,421 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=526 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/2c2b1f5daf6447809e7fae99d6be1edd 2024-11-07T12:43:18,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-07T12:43:18,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:18,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. as already flushing 2024-11-07T12:43:18,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:18,478 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:18,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:18,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:18,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-07T12:43:18,505 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/078ee72674144e07896f6c9773c9b62e is 50, key is test_row_0/C:col10/1730983397901/Put/seqid=0 2024-11-07T12:43:18,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742446_1622 (size=12301) 2024-11-07T12:43:18,568 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=526 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/078ee72674144e07896f6c9773c9b62e 2024-11-07T12:43:18,599 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/913d394be1b24fe5afe6a35eb45eaff4 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/913d394be1b24fe5afe6a35eb45eaff4 2024-11-07T12:43:18,622 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/913d394be1b24fe5afe6a35eb45eaff4, entries=200, sequenceid=526, filesize=14.4 K 2024-11-07T12:43:18,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/2c2b1f5daf6447809e7fae99d6be1edd as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/2c2b1f5daf6447809e7fae99d6be1edd 2024-11-07T12:43:18,633 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-07T12:43:18,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:18,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. as already flushing 2024-11-07T12:43:18,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:18,634 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:18,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:18,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:18,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:18,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41722 deadline: 1730983458633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:18,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:18,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:18,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:18,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:18,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41694 deadline: 1730983458633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:18,646 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:18,646 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:18,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:18,646 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:18,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41752 deadline: 1730983458633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:18,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:18,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1730983458633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:18,647 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:18,647 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:18,647 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:18,647 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:18,647 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:18,647 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:18,647 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/2c2b1f5daf6447809e7fae99d6be1edd, entries=150, sequenceid=526, filesize=12.0 K 2024-11-07T12:43:18,654 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/078ee72674144e07896f6c9773c9b62e as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/078ee72674144e07896f6c9773c9b62e 2024-11-07T12:43:18,678 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/078ee72674144e07896f6c9773c9b62e, entries=150, sequenceid=526, filesize=12.0 K 2024-11-07T12:43:18,684 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 850d939a0dd41b63931f314702af2ad5 in 777ms, sequenceid=526, compaction requested=true 2024-11-07T12:43:18,684 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 850d939a0dd41b63931f314702af2ad5: 2024-11-07T12:43:18,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 850d939a0dd41b63931f314702af2ad5:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:43:18,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:43:18,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 850d939a0dd41b63931f314702af2ad5:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:43:18,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-07T12:43:18,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 850d939a0dd41b63931f314702af2ad5:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:43:18,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-07T12:43:18,684 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:43:18,684 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:43:18,689 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:43:18,689 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 850d939a0dd41b63931f314702af2ad5/B is initiating minor compaction (all files) 2024-11-07T12:43:18,689 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 850d939a0dd41b63931f314702af2ad5/B in TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:18,689 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/c6673fc2cb4949459802fba9bb0a18fc, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/f048965776a14629a3f0364ffb697c9b, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/2c2b1f5daf6447809e7fae99d6be1edd] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp, totalSize=37.0 K 2024-11-07T12:43:18,691 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42771 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:43:18,691 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting c6673fc2cb4949459802fba9bb0a18fc, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=489, earliestPutTs=1730983397147 2024-11-07T12:43:18,691 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): 850d939a0dd41b63931f314702af2ad5/A is initiating minor compaction (all files) 2024-11-07T12:43:18,691 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 850d939a0dd41b63931f314702af2ad5/A in TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:18,691 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/f6ecd18bef2b40eab22598b4730223cd, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/0135c0b0b316427c890f239ac86d2d30, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/913d394be1b24fe5afe6a35eb45eaff4] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp, totalSize=41.8 K 2024-11-07T12:43:18,691 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting f048965776a14629a3f0364ffb697c9b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=511, earliestPutTs=1730983397619 2024-11-07T12:43:18,691 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c2b1f5daf6447809e7fae99d6be1edd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=526, earliestPutTs=1730983397775 2024-11-07T12:43:18,691 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6ecd18bef2b40eab22598b4730223cd, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=489, earliestPutTs=1730983397147 2024-11-07T12:43:18,692 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0135c0b0b316427c890f239ac86d2d30, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=511, earliestPutTs=1730983397600 2024-11-07T12:43:18,692 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 913d394be1b24fe5afe6a35eb45eaff4, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=526, earliestPutTs=1730983397774 2024-11-07T12:43:18,706 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 850d939a0dd41b63931f314702af2ad5#B#compaction#533 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:43:18,706 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/e46e742da3cd4cdb9e9a1f9bb0dae2da is 50, key is test_row_0/B:col10/1730983397901/Put/seqid=0 2024-11-07T12:43:18,713 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 850d939a0dd41b63931f314702af2ad5#A#compaction#534 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:43:18,714 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/6cb5181a93dd438baa46289b5dd4bcab is 50, key is test_row_0/A:col10/1730983397901/Put/seqid=0 2024-11-07T12:43:18,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742447_1623 (size=13391) 2024-11-07T12:43:18,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742448_1624 (size=13391) 2024-11-07T12:43:18,738 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/6cb5181a93dd438baa46289b5dd4bcab as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/6cb5181a93dd438baa46289b5dd4bcab 2024-11-07T12:43:18,747 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 850d939a0dd41b63931f314702af2ad5/A of 850d939a0dd41b63931f314702af2ad5 into 6cb5181a93dd438baa46289b5dd4bcab(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:43:18,747 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 850d939a0dd41b63931f314702af2ad5: 2024-11-07T12:43:18,747 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., storeName=850d939a0dd41b63931f314702af2ad5/A, priority=13, startTime=1730983398684; duration=0sec 2024-11-07T12:43:18,750 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:43:18,750 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 850d939a0dd41b63931f314702af2ad5:A 2024-11-07T12:43:18,751 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-07T12:43:18,755 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-07T12:43:18,755 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-07T12:43:18,755 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. because compaction request was cancelled 2024-11-07T12:43:18,755 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 850d939a0dd41b63931f314702af2ad5:C 2024-11-07T12:43:18,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-07T12:43:18,805 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-07T12:43:18,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:18,806 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2902): Flushing 850d939a0dd41b63931f314702af2ad5 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-07T12:43:18,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=A 2024-11-07T12:43:18,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:18,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=B 2024-11-07T12:43:18,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:18,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=C 2024-11-07T12:43:18,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:18,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/5fbac044cb6c4a349af35dd77582f0a6 is 50, key is test_row_0/A:col10/1730983397957/Put/seqid=0 2024-11-07T12:43:18,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742449_1625 (size=12301) 2024-11-07T12:43:18,828 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=545 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/5fbac044cb6c4a349af35dd77582f0a6 2024-11-07T12:43:18,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/320957b359d942f38310a60943fa6ad6 is 50, key is test_row_0/B:col10/1730983397957/Put/seqid=0 2024-11-07T12:43:18,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742450_1626 (size=12301) 2024-11-07T12:43:18,850 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=545 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/320957b359d942f38310a60943fa6ad6 2024-11-07T12:43:18,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/13692a90aac24bb1a276669a848dee7c is 50, key is test_row_0/C:col10/1730983397957/Put/seqid=0 2024-11-07T12:43:18,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742451_1627 (size=12301) 2024-11-07T12:43:19,130 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/e46e742da3cd4cdb9e9a1f9bb0dae2da as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/e46e742da3cd4cdb9e9a1f9bb0dae2da 2024-11-07T12:43:19,134 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 850d939a0dd41b63931f314702af2ad5/B of 850d939a0dd41b63931f314702af2ad5 into e46e742da3cd4cdb9e9a1f9bb0dae2da(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:43:19,134 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 850d939a0dd41b63931f314702af2ad5: 2024-11-07T12:43:19,134 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., storeName=850d939a0dd41b63931f314702af2ad5/B, priority=13, startTime=1730983398684; duration=0sec 2024-11-07T12:43:19,134 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:19,134 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 850d939a0dd41b63931f314702af2ad5:B 2024-11-07T12:43:19,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 850d939a0dd41b63931f314702af2ad5 2024-11-07T12:43:19,160 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. as already flushing 2024-11-07T12:43:19,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:19,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1730983459194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:19,205 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:19,205 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:19,205 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:19,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:19,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41694 deadline: 1730983459195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:19,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:19,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41722 deadline: 1730983459196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:19,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:19,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41752 deadline: 1730983459197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:19,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:19,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:19,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:19,214 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:19,214 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:19,214 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:19,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:19,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:19,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:19,299 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=545 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/13692a90aac24bb1a276669a848dee7c 2024-11-07T12:43:19,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-07T12:43:19,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/5fbac044cb6c4a349af35dd77582f0a6 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/5fbac044cb6c4a349af35dd77582f0a6 2024-11-07T12:43:19,307 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/5fbac044cb6c4a349af35dd77582f0a6, entries=150, sequenceid=545, filesize=12.0 K 2024-11-07T12:43:19,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/320957b359d942f38310a60943fa6ad6 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/320957b359d942f38310a60943fa6ad6 2024-11-07T12:43:19,311 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/320957b359d942f38310a60943fa6ad6, entries=150, sequenceid=545, filesize=12.0 K 2024-11-07T12:43:19,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/13692a90aac24bb1a276669a848dee7c as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/13692a90aac24bb1a276669a848dee7c 2024-11-07T12:43:19,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:19,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1730983459312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:19,315 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:19,315 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:19,315 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:19,316 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/13692a90aac24bb1a276669a848dee7c, entries=150, sequenceid=545, filesize=12.0 K 2024-11-07T12:43:19,317 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for 850d939a0dd41b63931f314702af2ad5 in 511ms, sequenceid=545, compaction requested=true 2024-11-07T12:43:19,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2603): Flush status journal for 850d939a0dd41b63931f314702af2ad5: 2024-11-07T12:43:19,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:19,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=142 2024-11-07T12:43:19,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4169): Remote procedure done, pid=142 2024-11-07T12:43:19,321 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-07T12:43:19,321 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1550 sec 2024-11-07T12:43:19,323 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees in 1.1610 sec 2024-11-07T12:43:19,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 850d939a0dd41b63931f314702af2ad5 2024-11-07T12:43:19,327 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 850d939a0dd41b63931f314702af2ad5 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-07T12:43:19,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=A 2024-11-07T12:43:19,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:19,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=B 2024-11-07T12:43:19,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:19,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=C 2024-11-07T12:43:19,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:19,332 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/a08a587b9e184fdfb06489626165931d is 50, key is test_row_0/A:col10/1730983399162/Put/seqid=0 2024-11-07T12:43:19,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742452_1628 (size=14741) 2024-11-07T12:43:19,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:19,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41694 deadline: 1730983459364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:19,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:19,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:19,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:19,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:19,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41752 deadline: 1730983459378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:19,380 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:19,380 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:19,380 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:19,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:19,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41722 deadline: 1730983459379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:19,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:19,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:19,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:19,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:19,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41752 deadline: 1730983459482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:19,492 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:19,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:19,493 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:19,493 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:19,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41694 deadline: 1730983459482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:19,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:19,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:19,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:19,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:19,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41722 deadline: 1730983459501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:19,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:19,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:19,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:19,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:19,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1730983459521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:19,531 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:19,531 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:19,532 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:19,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:19,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41752 deadline: 1730983459702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:19,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:19,708 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:19,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41694 deadline: 1730983459702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:19,708 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:19,708 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:19,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:19,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:19,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:19,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:19,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41722 deadline: 1730983459711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:19,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:19,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:19,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:19,744 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=563 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/a08a587b9e184fdfb06489626165931d 2024-11-07T12:43:19,770 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/60749c65e45843d98aa0ea6d4bf65fc7 is 50, key is test_row_0/B:col10/1730983399162/Put/seqid=0 2024-11-07T12:43:19,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742453_1629 (size=12301) 2024-11-07T12:43:19,779 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=563 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/60749c65e45843d98aa0ea6d4bf65fc7 2024-11-07T12:43:19,786 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/09a3391ebbfe4fe9bedb237326c36370 is 50, key is test_row_0/C:col10/1730983399162/Put/seqid=0 2024-11-07T12:43:19,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742454_1630 (size=12301) 2024-11-07T12:43:19,815 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=563 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/09a3391ebbfe4fe9bedb237326c36370 2024-11-07T12:43:19,819 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/a08a587b9e184fdfb06489626165931d as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/a08a587b9e184fdfb06489626165931d 2024-11-07T12:43:19,823 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/a08a587b9e184fdfb06489626165931d, entries=200, sequenceid=563, filesize=14.4 K 2024-11-07T12:43:19,824 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/60749c65e45843d98aa0ea6d4bf65fc7 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/60749c65e45843d98aa0ea6d4bf65fc7 2024-11-07T12:43:19,828 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/60749c65e45843d98aa0ea6d4bf65fc7, entries=150, sequenceid=563, filesize=12.0 K 2024-11-07T12:43:19,829 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/09a3391ebbfe4fe9bedb237326c36370 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/09a3391ebbfe4fe9bedb237326c36370 2024-11-07T12:43:19,833 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/09a3391ebbfe4fe9bedb237326c36370, entries=150, sequenceid=563, filesize=12.0 K 2024-11-07T12:43:19,834 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 850d939a0dd41b63931f314702af2ad5 in 507ms, sequenceid=563, compaction requested=true 2024-11-07T12:43:19,834 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 850d939a0dd41b63931f314702af2ad5: 2024-11-07T12:43:19,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 850d939a0dd41b63931f314702af2ad5:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:43:19,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:19,835 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:43:19,835 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:43:19,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 850d939a0dd41b63931f314702af2ad5:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:43:19,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:19,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 850d939a0dd41b63931f314702af2ad5:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:43:19,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:43:19,836 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40433 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:43:19,836 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): 850d939a0dd41b63931f314702af2ad5/A is initiating minor compaction (all files) 2024-11-07T12:43:19,836 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 850d939a0dd41b63931f314702af2ad5/A in TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:19,836 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/6cb5181a93dd438baa46289b5dd4bcab, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/5fbac044cb6c4a349af35dd77582f0a6, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/a08a587b9e184fdfb06489626165931d] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp, totalSize=39.5 K 2024-11-07T12:43:19,837 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:43:19,837 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 850d939a0dd41b63931f314702af2ad5/B is initiating minor compaction (all files) 2024-11-07T12:43:19,837 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 850d939a0dd41b63931f314702af2ad5/B in TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:19,837 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/e46e742da3cd4cdb9e9a1f9bb0dae2da, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/320957b359d942f38310a60943fa6ad6, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/60749c65e45843d98aa0ea6d4bf65fc7] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp, totalSize=37.1 K 2024-11-07T12:43:19,837 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6cb5181a93dd438baa46289b5dd4bcab, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=526, earliestPutTs=1730983397775 2024-11-07T12:43:19,838 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting e46e742da3cd4cdb9e9a1f9bb0dae2da, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=526, earliestPutTs=1730983397775 2024-11-07T12:43:19,838 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5fbac044cb6c4a349af35dd77582f0a6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=545, earliestPutTs=1730983397934 2024-11-07T12:43:19,839 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 320957b359d942f38310a60943fa6ad6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=545, earliestPutTs=1730983397934 2024-11-07T12:43:19,839 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting a08a587b9e184fdfb06489626165931d, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=563, earliestPutTs=1730983399162 2024-11-07T12:43:19,839 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 60749c65e45843d98aa0ea6d4bf65fc7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=563, earliestPutTs=1730983399162 2024-11-07T12:43:19,855 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 850d939a0dd41b63931f314702af2ad5#A#compaction#541 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:43:19,856 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/7a7e459216a7464ea5d9e7f575fd486a is 50, key is test_row_0/A:col10/1730983399162/Put/seqid=0 2024-11-07T12:43:19,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 850d939a0dd41b63931f314702af2ad5 2024-11-07T12:43:19,862 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 850d939a0dd41b63931f314702af2ad5 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-07T12:43:19,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=A 2024-11-07T12:43:19,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:19,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=B 2024-11-07T12:43:19,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:19,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=C 2024-11-07T12:43:19,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:19,875 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 850d939a0dd41b63931f314702af2ad5#B#compaction#542 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:43:19,876 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/cee0db2678384ef4b62d7f2709a95cf0 is 50, key is test_row_0/B:col10/1730983399162/Put/seqid=0 2024-11-07T12:43:19,882 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/2d2241e109b54637a131dcd1e5775411 is 50, key is test_row_0/A:col10/1730983399850/Put/seqid=0 2024-11-07T12:43:19,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742456_1632 (size=13493) 2024-11-07T12:43:19,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742455_1631 (size=13493) 2024-11-07T12:43:19,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742457_1633 (size=14741) 2024-11-07T12:43:19,945 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=579 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/2d2241e109b54637a131dcd1e5775411 2024-11-07T12:43:19,958 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/7a7e459216a7464ea5d9e7f575fd486a as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/7a7e459216a7464ea5d9e7f575fd486a 2024-11-07T12:43:19,966 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/cee0db2678384ef4b62d7f2709a95cf0 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/cee0db2678384ef4b62d7f2709a95cf0 2024-11-07T12:43:19,967 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/7f7a27ae46da43b5b0a5bbf60e5734ba is 50, key is test_row_0/B:col10/1730983399850/Put/seqid=0 2024-11-07T12:43:19,969 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 850d939a0dd41b63931f314702af2ad5/A of 850d939a0dd41b63931f314702af2ad5 into 7a7e459216a7464ea5d9e7f575fd486a(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:43:19,969 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 850d939a0dd41b63931f314702af2ad5: 2024-11-07T12:43:19,969 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., storeName=850d939a0dd41b63931f314702af2ad5/A, priority=13, startTime=1730983399834; duration=0sec 2024-11-07T12:43:19,969 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:43:19,969 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 850d939a0dd41b63931f314702af2ad5:A 2024-11-07T12:43:19,969 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:43:19,971 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50192 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:43:19,971 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): 850d939a0dd41b63931f314702af2ad5/C is initiating minor compaction (all files) 2024-11-07T12:43:19,971 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 850d939a0dd41b63931f314702af2ad5/C in TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:19,971 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/0626691b740e49b69a54da0b8fe7eeb7, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/078ee72674144e07896f6c9773c9b62e, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/13692a90aac24bb1a276669a848dee7c, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/09a3391ebbfe4fe9bedb237326c36370] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp, totalSize=49.0 K 2024-11-07T12:43:19,972 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0626691b740e49b69a54da0b8fe7eeb7, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=511, earliestPutTs=1730983397619 2024-11-07T12:43:19,973 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 078ee72674144e07896f6c9773c9b62e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=526, earliestPutTs=1730983397775 2024-11-07T12:43:19,974 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13692a90aac24bb1a276669a848dee7c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=545, earliestPutTs=1730983397934 2024-11-07T12:43:19,974 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 850d939a0dd41b63931f314702af2ad5/B of 850d939a0dd41b63931f314702af2ad5 into cee0db2678384ef4b62d7f2709a95cf0(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:43:19,974 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 850d939a0dd41b63931f314702af2ad5: 2024-11-07T12:43:19,974 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., storeName=850d939a0dd41b63931f314702af2ad5/B, priority=13, startTime=1730983399835; duration=0sec 2024-11-07T12:43:19,974 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:19,974 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 850d939a0dd41b63931f314702af2ad5:B 2024-11-07T12:43:19,975 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09a3391ebbfe4fe9bedb237326c36370, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=563, earliestPutTs=1730983399162 2024-11-07T12:43:19,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742458_1634 (size=12301) 2024-11-07T12:43:19,981 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=579 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/7f7a27ae46da43b5b0a5bbf60e5734ba 2024-11-07T12:43:19,995 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 850d939a0dd41b63931f314702af2ad5#C#compaction#545 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:43:19,996 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/38de3ee681e24ddeac6959751bd7b842 is 50, key is test_row_0/C:col10/1730983399162/Put/seqid=0 2024-11-07T12:43:20,001 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/7ee0db856c7b4cacae2c7e875e6678f5 is 50, key is test_row_0/C:col10/1730983399850/Put/seqid=0 2024-11-07T12:43:20,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742459_1635 (size=13425) 2024-11-07T12:43:20,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742460_1636 (size=12301) 2024-11-07T12:43:20,013 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=579 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/7ee0db856c7b4cacae2c7e875e6678f5 2024-11-07T12:43:20,017 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/2d2241e109b54637a131dcd1e5775411 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/2d2241e109b54637a131dcd1e5775411 2024-11-07T12:43:20,020 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/2d2241e109b54637a131dcd1e5775411, entries=200, sequenceid=579, filesize=14.4 K 2024-11-07T12:43:20,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/7f7a27ae46da43b5b0a5bbf60e5734ba as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/7f7a27ae46da43b5b0a5bbf60e5734ba 2024-11-07T12:43:20,024 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/7f7a27ae46da43b5b0a5bbf60e5734ba, entries=150, sequenceid=579, filesize=12.0 K 2024-11-07T12:43:20,025 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/7ee0db856c7b4cacae2c7e875e6678f5 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/7ee0db856c7b4cacae2c7e875e6678f5 2024-11-07T12:43:20,031 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/7ee0db856c7b4cacae2c7e875e6678f5, entries=150, sequenceid=579, filesize=12.0 K 2024-11-07T12:43:20,032 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=80.51 KB/82440 for 850d939a0dd41b63931f314702af2ad5 in 170ms, sequenceid=579, compaction requested=false 2024-11-07T12:43:20,032 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 850d939a0dd41b63931f314702af2ad5: 2024-11-07T12:43:20,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 850d939a0dd41b63931f314702af2ad5 2024-11-07T12:43:20,040 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 850d939a0dd41b63931f314702af2ad5 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-07T12:43:20,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=A 2024-11-07T12:43:20,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:20,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=B 2024-11-07T12:43:20,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:20,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=C 2024-11-07T12:43:20,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:20,045 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/66cbac03ea85465b9d8cb240691004b0 is 50, key is test_row_0/A:col10/1730983399981/Put/seqid=0 2024-11-07T12:43:20,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742461_1637 (size=14741) 2024-11-07T12:43:20,061 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=597 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/66cbac03ea85465b9d8cb240691004b0 2024-11-07T12:43:20,068 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/068e892d41074dc0b236a325c9be0db3 is 50, key is test_row_0/B:col10/1730983399981/Put/seqid=0 2024-11-07T12:43:20,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742462_1638 (size=12301) 2024-11-07T12:43:20,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:20,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41694 deadline: 1730983460082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:20,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:20,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41722 deadline: 1730983460083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:20,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:20,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:20,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:20,109 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:20,109 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:20,109 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:20,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:20,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41752 deadline: 1730983460107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:20,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:20,124 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:20,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1730983460109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:20,124 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:20,124 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:20,124 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:20,124 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:20,125 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:20,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:20,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41694 deadline: 1730983460213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:20,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:20,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41722 deadline: 1730983460213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:20,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:20,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:20,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:20,224 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:20,224 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:20,224 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:20,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:20,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1730983460232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:20,241 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:20,241 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:20,241 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:20,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:20,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41752 deadline: 1730983460232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:20,243 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:20,243 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:20,243 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:20,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-07T12:43:20,312 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:43:20,312 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:43:20,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees 2024-11-07T12:43:20,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-07T12:43:20,319 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=143, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:43:20,323 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=143, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:43:20,323 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:43:20,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-07T12:43:20,433 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/38de3ee681e24ddeac6959751bd7b842 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/38de3ee681e24ddeac6959751bd7b842 2024-11-07T12:43:20,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:20,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41694 deadline: 1730983460434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:20,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:20,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:20,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:20,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:20,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41722 deadline: 1730983460437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:20,448 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:20,448 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:20,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:20,455 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 850d939a0dd41b63931f314702af2ad5/C of 850d939a0dd41b63931f314702af2ad5 into 38de3ee681e24ddeac6959751bd7b842(size=13.1 K), total size for store is 25.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:43:20,455 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 850d939a0dd41b63931f314702af2ad5: 2024-11-07T12:43:20,455 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., storeName=850d939a0dd41b63931f314702af2ad5/C, priority=12, startTime=1730983399835; duration=0sec 2024-11-07T12:43:20,455 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:20,455 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 850d939a0dd41b63931f314702af2ad5:C 2024-11-07T12:43:20,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:20,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1730983460452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:20,463 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:20,463 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:20,463 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:20,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:20,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41752 deadline: 1730983460453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:20,467 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:20,467 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:20,467 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:20,497 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-07T12:43:20,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:20,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. as already flushing 2024-11-07T12:43:20,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:20,497 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:20,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:20,501 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=597 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/068e892d41074dc0b236a325c9be0db3 2024-11-07T12:43:20,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:20,518 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/c8b3e751295b423793bf15f6b2eccc0a is 50, key is test_row_0/C:col10/1730983399981/Put/seqid=0 2024-11-07T12:43:20,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742463_1639 (size=12301) 2024-11-07T12:43:20,528 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=597 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/c8b3e751295b423793bf15f6b2eccc0a 2024-11-07T12:43:20,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/66cbac03ea85465b9d8cb240691004b0 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/66cbac03ea85465b9d8cb240691004b0 2024-11-07T12:43:20,542 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/66cbac03ea85465b9d8cb240691004b0, entries=200, sequenceid=597, filesize=14.4 K 2024-11-07T12:43:20,542 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/068e892d41074dc0b236a325c9be0db3 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/068e892d41074dc0b236a325c9be0db3 2024-11-07T12:43:20,545 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/068e892d41074dc0b236a325c9be0db3, entries=150, sequenceid=597, filesize=12.0 K 2024-11-07T12:43:20,546 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/c8b3e751295b423793bf15f6b2eccc0a as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/c8b3e751295b423793bf15f6b2eccc0a 2024-11-07T12:43:20,550 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/c8b3e751295b423793bf15f6b2eccc0a, entries=150, sequenceid=597, filesize=12.0 K 2024-11-07T12:43:20,551 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 850d939a0dd41b63931f314702af2ad5 in 511ms, sequenceid=597, compaction requested=true 2024-11-07T12:43:20,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 850d939a0dd41b63931f314702af2ad5: 2024-11-07T12:43:20,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 850d939a0dd41b63931f314702af2ad5:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:43:20,551 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:43:20,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:20,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 850d939a0dd41b63931f314702af2ad5:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:43:20,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:43:20,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 850d939a0dd41b63931f314702af2ad5:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:43:20,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-07T12:43:20,552 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42975 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:43:20,552 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): 850d939a0dd41b63931f314702af2ad5/A is initiating minor compaction (all files) 2024-11-07T12:43:20,552 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 850d939a0dd41b63931f314702af2ad5/A in TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:20,552 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/7a7e459216a7464ea5d9e7f575fd486a, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/2d2241e109b54637a131dcd1e5775411, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/66cbac03ea85465b9d8cb240691004b0] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp, totalSize=42.0 K 2024-11-07T12:43:20,552 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:43:20,552 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a7e459216a7464ea5d9e7f575fd486a, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=563, earliestPutTs=1730983399162 2024-11-07T12:43:20,553 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d2241e109b54637a131dcd1e5775411, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=579, earliestPutTs=1730983399362 2024-11-07T12:43:20,553 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66cbac03ea85465b9d8cb240691004b0, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=597, earliestPutTs=1730983399977 2024-11-07T12:43:20,553 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38095 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:43:20,553 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 850d939a0dd41b63931f314702af2ad5/B is initiating minor compaction (all files) 2024-11-07T12:43:20,553 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 850d939a0dd41b63931f314702af2ad5/B in TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:20,554 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/cee0db2678384ef4b62d7f2709a95cf0, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/7f7a27ae46da43b5b0a5bbf60e5734ba, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/068e892d41074dc0b236a325c9be0db3] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp, totalSize=37.2 K 2024-11-07T12:43:20,554 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting cee0db2678384ef4b62d7f2709a95cf0, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=563, earliestPutTs=1730983399162 2024-11-07T12:43:20,554 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f7a27ae46da43b5b0a5bbf60e5734ba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=579, earliestPutTs=1730983399376 2024-11-07T12:43:20,555 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 068e892d41074dc0b236a325c9be0db3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=597, earliestPutTs=1730983399977 2024-11-07T12:43:20,560 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 850d939a0dd41b63931f314702af2ad5#A#compaction#550 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:43:20,561 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/64f0056f35134016b346341eb7f32757 is 50, key is test_row_0/A:col10/1730983399981/Put/seqid=0 2024-11-07T12:43:20,570 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 850d939a0dd41b63931f314702af2ad5#B#compaction#551 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:43:20,570 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/5f4da1bf5ee143b1ba8bddc4b76db26e is 50, key is test_row_0/B:col10/1730983399981/Put/seqid=0 2024-11-07T12:43:20,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742464_1640 (size=13595) 2024-11-07T12:43:20,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742465_1641 (size=13595) 2024-11-07T12:43:20,609 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/64f0056f35134016b346341eb7f32757 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/64f0056f35134016b346341eb7f32757 2024-11-07T12:43:20,614 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/5f4da1bf5ee143b1ba8bddc4b76db26e as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/5f4da1bf5ee143b1ba8bddc4b76db26e 2024-11-07T12:43:20,616 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 850d939a0dd41b63931f314702af2ad5/A of 850d939a0dd41b63931f314702af2ad5 into 64f0056f35134016b346341eb7f32757(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:43:20,616 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 850d939a0dd41b63931f314702af2ad5: 2024-11-07T12:43:20,616 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., storeName=850d939a0dd41b63931f314702af2ad5/A, priority=13, startTime=1730983400551; duration=0sec 2024-11-07T12:43:20,616 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:43:20,616 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 850d939a0dd41b63931f314702af2ad5:A 2024-11-07T12:43:20,616 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:43:20,617 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38027 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:43:20,617 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): 850d939a0dd41b63931f314702af2ad5/C is initiating minor compaction (all files) 2024-11-07T12:43:20,617 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 850d939a0dd41b63931f314702af2ad5/C in TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:20,617 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/38de3ee681e24ddeac6959751bd7b842, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/7ee0db856c7b4cacae2c7e875e6678f5, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/c8b3e751295b423793bf15f6b2eccc0a] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp, totalSize=37.1 K 2024-11-07T12:43:20,618 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38de3ee681e24ddeac6959751bd7b842, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=563, earliestPutTs=1730983399162 2024-11-07T12:43:20,619 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 850d939a0dd41b63931f314702af2ad5/B of 850d939a0dd41b63931f314702af2ad5 into 5f4da1bf5ee143b1ba8bddc4b76db26e(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:43:20,619 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 850d939a0dd41b63931f314702af2ad5: 2024-11-07T12:43:20,619 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., storeName=850d939a0dd41b63931f314702af2ad5/B, priority=13, startTime=1730983400552; duration=0sec 2024-11-07T12:43:20,619 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:20,619 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 850d939a0dd41b63931f314702af2ad5:B 2024-11-07T12:43:20,621 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ee0db856c7b4cacae2c7e875e6678f5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=579, earliestPutTs=1730983399376 2024-11-07T12:43:20,624 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting c8b3e751295b423793bf15f6b2eccc0a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=597, earliestPutTs=1730983399977 2024-11-07T12:43:20,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-07T12:43:20,656 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 850d939a0dd41b63931f314702af2ad5#C#compaction#552 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:43:20,657 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/2578e9695153461db5ed80b84c4d5a24 is 50, key is test_row_0/C:col10/1730983399981/Put/seqid=0 2024-11-07T12:43:20,659 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-07T12:43:20,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:20,660 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2902): Flushing 850d939a0dd41b63931f314702af2ad5 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-07T12:43:20,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=A 2024-11-07T12:43:20,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:20,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=B 2024-11-07T12:43:20,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:20,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=C 2024-11-07T12:43:20,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:20,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/5682fc0ac8e6470bbb8f859c8c0fdb3a is 50, key is test_row_0/A:col10/1730983400104/Put/seqid=0 2024-11-07T12:43:20,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742466_1642 (size=13527) 2024-11-07T12:43:20,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742467_1643 (size=12301) 2024-11-07T12:43:20,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 850d939a0dd41b63931f314702af2ad5 2024-11-07T12:43:20,757 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. as already flushing 2024-11-07T12:43:20,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:20,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41722 deadline: 1730983460785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:20,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:20,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:20,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:20,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:20,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41694 deadline: 1730983460788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:20,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:20,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1730983460795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:20,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:20,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:20,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:20,802 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:20,802 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:20,802 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:20,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:20,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41752 deadline: 1730983460796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:20,810 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:20,810 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:20,810 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:20,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:20,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41722 deadline: 1730983460901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:20,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:20,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:20,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:20,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:20,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1730983460912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:20,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:20,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41694 deadline: 1730983460916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:20,924 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:20,924 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:20,925 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:20,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:20,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:20,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:20,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:20,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41752 deadline: 1730983460922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:20,932 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:20,932 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:20,932 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:20,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-07T12:43:21,083 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/2578e9695153461db5ed80b84c4d5a24 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/2578e9695153461db5ed80b84c4d5a24 2024-11-07T12:43:21,094 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 850d939a0dd41b63931f314702af2ad5/C of 850d939a0dd41b63931f314702af2ad5 into 2578e9695153461db5ed80b84c4d5a24(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:43:21,094 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 850d939a0dd41b63931f314702af2ad5: 2024-11-07T12:43:21,094 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., storeName=850d939a0dd41b63931f314702af2ad5/C, priority=13, startTime=1730983400552; duration=0sec 2024-11-07T12:43:21,094 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:21,094 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 850d939a0dd41b63931f314702af2ad5:C 2024-11-07T12:43:21,095 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=616 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/5682fc0ac8e6470bbb8f859c8c0fdb3a 2024-11-07T12:43:21,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/72009bec59f947f3bb234442ed6316c0 is 50, key is test_row_0/B:col10/1730983400104/Put/seqid=0 2024-11-07T12:43:21,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742468_1644 (size=12301) 2024-11-07T12:43:21,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:21,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41722 deadline: 1730983461122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:21,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:21,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:21,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:21,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:21,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1730983461133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:21,144 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:21,144 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:21,144 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:21,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:21,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41694 deadline: 1730983461135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:21,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:21,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:21,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:21,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:21,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41752 deadline: 1730983461142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:21,152 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:21,152 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:21,152 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:21,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:21,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41722 deadline: 1730983461442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:21,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:21,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:21,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:21,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-07T12:43:21,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:21,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1730983461451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:21,461 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:21,461 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:21,461 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:21,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:21,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41694 deadline: 1730983461451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:21,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:21,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41752 deadline: 1730983461462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:21,464 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:21,464 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:21,464 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:21,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:21,466 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:21,466 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:21,528 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=616 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/72009bec59f947f3bb234442ed6316c0 2024-11-07T12:43:21,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/2880b70564cd4f5e8dc436ad40a34a36 is 50, key is test_row_0/C:col10/1730983400104/Put/seqid=0 2024-11-07T12:43:21,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742469_1645 (size=12301) 2024-11-07T12:43:21,665 INFO [Thread-2343 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2343. 2024-11-07T12:43:21,665 DEBUG [Thread-2343 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:43:21,665 DEBUG [Thread-2343 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:43:21,667 INFO [Thread-2341 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2341. 2024-11-07T12:43:21,667 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:43:21,667 DEBUG [Thread-2341 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:43:21,667 DEBUG [Thread-2341 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:43:21,667 DEBUG [Thread-2343 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:43:21,668 DEBUG [Thread-2341 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:43:21,668 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:43:21,673 INFO [Thread-2345 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2345. 2024-11-07T12:43:21,673 DEBUG [Thread-2345 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:43:21,673 DEBUG [Thread-2345 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:43:21,673 DEBUG [Thread-2345 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:43:21,673 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:43:21,677 INFO [Thread-2339 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2339. 2024-11-07T12:43:21,677 DEBUG [Thread-2339 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:43:21,677 DEBUG [Thread-2339 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:43:21,677 DEBUG [Thread-2339 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:43:21,677 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:43:21,679 INFO [Thread-2337 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2337. 2024-11-07T12:43:21,679 DEBUG [Thread-2337 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:43:21,679 DEBUG [Thread-2337 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:43:21,679 DEBUG [Thread-2337 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:43:21,679 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:43:21,948 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=616 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/2880b70564cd4f5e8dc436ad40a34a36 2024-11-07T12:43:21,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/5682fc0ac8e6470bbb8f859c8c0fdb3a as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/5682fc0ac8e6470bbb8f859c8c0fdb3a 2024-11-07T12:43:21,955 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/5682fc0ac8e6470bbb8f859c8c0fdb3a, entries=150, sequenceid=616, filesize=12.0 K 2024-11-07T12:43:21,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/72009bec59f947f3bb234442ed6316c0 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/72009bec59f947f3bb234442ed6316c0 2024-11-07T12:43:21,959 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/72009bec59f947f3bb234442ed6316c0, entries=150, sequenceid=616, filesize=12.0 K 2024-11-07T12:43:21,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/2880b70564cd4f5e8dc436ad40a34a36 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/2880b70564cd4f5e8dc436ad40a34a36 2024-11-07T12:43:21,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:21,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41722 deadline: 1730983461962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:21,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 , the old value is region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:21,962 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/2880b70564cd4f5e8dc436ad40a34a36, entries=150, sequenceid=616, filesize=12.0 K 2024-11-07T12:43:21,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:21,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:43:21,963 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 850d939a0dd41b63931f314702af2ad5 in 1303ms, sequenceid=616, compaction requested=false 2024-11-07T12:43:21,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2603): Flush status journal for 850d939a0dd41b63931f314702af2ad5: 2024-11-07T12:43:21,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:21,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=144 2024-11-07T12:43:21,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4169): Remote procedure done, pid=144 2024-11-07T12:43:21,966 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-07T12:43:21,966 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6420 sec 2024-11-07T12:43:21,968 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees in 1.6550 sec 2024-11-07T12:43:21,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 850d939a0dd41b63931f314702af2ad5 2024-11-07T12:43:21,974 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 850d939a0dd41b63931f314702af2ad5 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-07T12:43:21,974 INFO [Thread-2326 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2326. 2024-11-07T12:43:21,974 INFO [Thread-2334 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2334. 2024-11-07T12:43:21,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=A 2024-11-07T12:43:21,974 DEBUG [Thread-2334 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:43:21,974 DEBUG [Thread-2326 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:43:21,974 DEBUG [Thread-2334 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:43:21,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:21,974 DEBUG [Thread-2326 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:43:21,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=B 2024-11-07T12:43:21,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:21,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=C 2024-11-07T12:43:21,974 DEBUG [Thread-2334 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:43:21,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:21,974 DEBUG [Thread-2326 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:43:21,974 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:43:21,975 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:43:21,976 INFO [Thread-2332 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2332. 2024-11-07T12:43:21,976 DEBUG [Thread-2332 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:43:21,976 DEBUG [Thread-2332 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:43:21,976 DEBUG [Thread-2332 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:43:21,976 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:43:21,979 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/f60b4f7e611b487aaa6725e12d266e0c is 50, key is test_row_0/A:col10/1730983400779/Put/seqid=0 2024-11-07T12:43:21,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742470_1646 (size=12301) 2024-11-07T12:43:22,387 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=636 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/f60b4f7e611b487aaa6725e12d266e0c 2024-11-07T12:43:22,394 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/79c1549d59ca402799e8e6d367fa4610 is 50, key is test_row_0/B:col10/1730983400779/Put/seqid=0 2024-11-07T12:43:22,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742471_1647 (size=12301) 2024-11-07T12:43:22,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-07T12:43:22,463 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:43:22,799 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=636 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/79c1549d59ca402799e8e6d367fa4610 2024-11-07T12:43:22,807 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/f94e9ed0d5a745df91c61521d90dc83b is 50, key is test_row_0/C:col10/1730983400779/Put/seqid=0 2024-11-07T12:43:22,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742472_1648 (size=12301) 2024-11-07T12:43:22,974 INFO [Thread-2330 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2330. 2024-11-07T12:43:22,974 DEBUG [Thread-2330 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:43:22,974 DEBUG [Thread-2330 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:43:22,974 DEBUG [Thread-2330 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:43:22,974 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:43:23,215 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=636 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/f94e9ed0d5a745df91c61521d90dc83b 2024-11-07T12:43:23,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/f60b4f7e611b487aaa6725e12d266e0c as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/f60b4f7e611b487aaa6725e12d266e0c 2024-11-07T12:43:23,222 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/f60b4f7e611b487aaa6725e12d266e0c, entries=150, sequenceid=636, filesize=12.0 K 2024-11-07T12:43:23,222 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/79c1549d59ca402799e8e6d367fa4610 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/79c1549d59ca402799e8e6d367fa4610 2024-11-07T12:43:23,225 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/79c1549d59ca402799e8e6d367fa4610, entries=150, sequenceid=636, filesize=12.0 K 2024-11-07T12:43:23,225 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/f94e9ed0d5a745df91c61521d90dc83b as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/f94e9ed0d5a745df91c61521d90dc83b 2024-11-07T12:43:23,227 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/f94e9ed0d5a745df91c61521d90dc83b, entries=150, sequenceid=636, filesize=12.0 K 2024-11-07T12:43:23,228 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=6.71 KB/6870 for 850d939a0dd41b63931f314702af2ad5 in 1254ms, sequenceid=636, compaction requested=true 2024-11-07T12:43:23,228 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 850d939a0dd41b63931f314702af2ad5: 2024-11-07T12:43:23,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 850d939a0dd41b63931f314702af2ad5:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:43:23,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:23,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 850d939a0dd41b63931f314702af2ad5:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:43:23,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:23,228 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:43:23,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 850d939a0dd41b63931f314702af2ad5:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:43:23,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:43:23,228 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:43:23,229 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38197 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:43:23,229 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): 850d939a0dd41b63931f314702af2ad5/A is initiating minor compaction (all files) 2024-11-07T12:43:23,229 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38197 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:43:23,229 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 850d939a0dd41b63931f314702af2ad5/A in TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:23,229 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 850d939a0dd41b63931f314702af2ad5/B is initiating minor compaction (all files) 2024-11-07T12:43:23,229 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 850d939a0dd41b63931f314702af2ad5/B in TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:23,229 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/64f0056f35134016b346341eb7f32757, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/5682fc0ac8e6470bbb8f859c8c0fdb3a, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/f60b4f7e611b487aaa6725e12d266e0c] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp, totalSize=37.3 K 2024-11-07T12:43:23,229 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/5f4da1bf5ee143b1ba8bddc4b76db26e, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/72009bec59f947f3bb234442ed6316c0, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/79c1549d59ca402799e8e6d367fa4610] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp, totalSize=37.3 K 2024-11-07T12:43:23,229 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f4da1bf5ee143b1ba8bddc4b76db26e, keycount=150, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=597, earliestPutTs=1730983399977 2024-11-07T12:43:23,229 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64f0056f35134016b346341eb7f32757, keycount=150, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=597, earliestPutTs=1730983399977 2024-11-07T12:43:23,230 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 72009bec59f947f3bb234442ed6316c0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=616, earliestPutTs=1730983400080 2024-11-07T12:43:23,230 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5682fc0ac8e6470bbb8f859c8c0fdb3a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=616, earliestPutTs=1730983400080 2024-11-07T12:43:23,230 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 79c1549d59ca402799e8e6d367fa4610, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=636, earliestPutTs=1730983400779 2024-11-07T12:43:23,230 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting f60b4f7e611b487aaa6725e12d266e0c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=636, earliestPutTs=1730983400779 2024-11-07T12:43:23,236 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 850d939a0dd41b63931f314702af2ad5#B#compaction#560 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:43:23,237 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/b6d333ee3895438bb5dae0a1d04a29c3 is 50, key is test_row_0/B:col10/1730983400779/Put/seqid=0 2024-11-07T12:43:23,237 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 850d939a0dd41b63931f314702af2ad5#A#compaction#559 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:43:23,237 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/a32b8c4d019e43cbaf7a0a803ad5eac1 is 50, key is test_row_0/A:col10/1730983400779/Put/seqid=0 2024-11-07T12:43:23,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742473_1649 (size=13697) 2024-11-07T12:43:23,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742474_1650 (size=13697) 2024-11-07T12:43:23,244 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/a32b8c4d019e43cbaf7a0a803ad5eac1 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/a32b8c4d019e43cbaf7a0a803ad5eac1 2024-11-07T12:43:23,247 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 850d939a0dd41b63931f314702af2ad5/A of 850d939a0dd41b63931f314702af2ad5 into a32b8c4d019e43cbaf7a0a803ad5eac1(size=13.4 K), total size for store is 13.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:43:23,247 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 850d939a0dd41b63931f314702af2ad5: 2024-11-07T12:43:23,247 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., storeName=850d939a0dd41b63931f314702af2ad5/A, priority=13, startTime=1730983403228; duration=0sec 2024-11-07T12:43:23,248 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:43:23,248 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 850d939a0dd41b63931f314702af2ad5:A 2024-11-07T12:43:23,248 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:43:23,248 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38129 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:43:23,248 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): 850d939a0dd41b63931f314702af2ad5/C is initiating minor compaction (all files) 2024-11-07T12:43:23,248 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 850d939a0dd41b63931f314702af2ad5/C in TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:23,249 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/2578e9695153461db5ed80b84c4d5a24, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/2880b70564cd4f5e8dc436ad40a34a36, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/f94e9ed0d5a745df91c61521d90dc83b] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp, totalSize=37.2 K 2024-11-07T12:43:23,249 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2578e9695153461db5ed80b84c4d5a24, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=597, earliestPutTs=1730983399977 2024-11-07T12:43:23,249 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2880b70564cd4f5e8dc436ad40a34a36, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=616, earliestPutTs=1730983400080 2024-11-07T12:43:23,249 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting f94e9ed0d5a745df91c61521d90dc83b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=636, earliestPutTs=1730983400779 2024-11-07T12:43:23,255 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 850d939a0dd41b63931f314702af2ad5#C#compaction#561 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:43:23,256 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/6973784b88934758b0506601aac4c118 is 50, key is test_row_0/C:col10/1730983400779/Put/seqid=0 2024-11-07T12:43:23,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742475_1651 (size=13629) 2024-11-07T12:43:23,644 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/b6d333ee3895438bb5dae0a1d04a29c3 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/b6d333ee3895438bb5dae0a1d04a29c3 2024-11-07T12:43:23,648 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 850d939a0dd41b63931f314702af2ad5/B of 850d939a0dd41b63931f314702af2ad5 into b6d333ee3895438bb5dae0a1d04a29c3(size=13.4 K), total size for store is 13.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:43:23,648 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 850d939a0dd41b63931f314702af2ad5: 2024-11-07T12:43:23,648 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., storeName=850d939a0dd41b63931f314702af2ad5/B, priority=13, startTime=1730983403228; duration=0sec 2024-11-07T12:43:23,648 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:23,648 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 850d939a0dd41b63931f314702af2ad5:B 2024-11-07T12:43:23,662 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/6973784b88934758b0506601aac4c118 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/6973784b88934758b0506601aac4c118 2024-11-07T12:43:23,665 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 850d939a0dd41b63931f314702af2ad5/C of 850d939a0dd41b63931f314702af2ad5 into 6973784b88934758b0506601aac4c118(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:43:23,666 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 850d939a0dd41b63931f314702af2ad5: 2024-11-07T12:43:23,666 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., storeName=850d939a0dd41b63931f314702af2ad5/C, priority=13, startTime=1730983403228; duration=0sec 2024-11-07T12:43:23,666 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:23,666 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 850d939a0dd41b63931f314702af2ad5:C 2024-11-07T12:43:26,073 INFO [Thread-2328 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2328. 2024-11-07T12:43:26,073 DEBUG [Thread-2328 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:43:26,074 DEBUG [Thread-2328 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:43:26,074 DEBUG [Thread-2328 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:43:26,074 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:43:26,074 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-07T12:43:26,074 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 86 2024-11-07T12:43:26,074 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 55 2024-11-07T12:43:26,074 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 168 2024-11-07T12:43:26,074 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 130 2024-11-07T12:43:26,074 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 77 2024-11-07T12:43:26,074 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-07T12:43:26,074 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-07T12:43:26,074 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 849 2024-11-07T12:43:26,074 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2547 rows 2024-11-07T12:43:26,074 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 847 2024-11-07T12:43:26,074 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2541 rows 2024-11-07T12:43:26,074 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 846 2024-11-07T12:43:26,074 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2533 rows 2024-11-07T12:43:26,074 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 838 2024-11-07T12:43:26,074 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2514 rows 2024-11-07T12:43:26,074 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 844 2024-11-07T12:43:26,074 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2532 rows 2024-11-07T12:43:26,074 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-07T12:43:26,074 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T12:43:26,074 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:43:26,074 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:43:26,074 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:43:26,075 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-07T12:43:26,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-07T12:43:26,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-07T12:43:26,078 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730983406078"}]},"ts":"1730983406078"} 2024-11-07T12:43:26,080 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-07T12:43:26,080 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-07T12:43:26,080 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-07T12:43:26,082 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=850d939a0dd41b63931f314702af2ad5, UNASSIGN}] 2024-11-07T12:43:26,083 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=850d939a0dd41b63931f314702af2ad5, UNASSIGN 2024-11-07T12:43:26,083 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=850d939a0dd41b63931f314702af2ad5, regionState=CLOSING, regionLocation=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:26,085 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=850d939a0dd41b63931f314702af2ad5, UNASSIGN because future has completed 2024-11-07T12:43:26,085 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-07T12:43:26,085 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE, hasLock=false; CloseRegionProcedure 850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217}] 2024-11-07T12:43:26,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-07T12:43:26,237 INFO [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(122): Close 850d939a0dd41b63931f314702af2ad5 2024-11-07T12:43:26,237 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-07T12:43:26,238 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1722): Closing 850d939a0dd41b63931f314702af2ad5, disabling compactions & flushes 2024-11-07T12:43:26,238 INFO [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:26,238 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:26,238 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. after waiting 0 ms 2024-11-07T12:43:26,238 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:26,238 INFO [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(2902): Flushing 850d939a0dd41b63931f314702af2ad5 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-11-07T12:43:26,238 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=A 2024-11-07T12:43:26,238 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:26,238 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=B 2024-11-07T12:43:26,238 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:26,238 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 850d939a0dd41b63931f314702af2ad5, store=C 2024-11-07T12:43:26,238 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:26,241 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/fafe66263b3844ba87584d0bc79a51d3 is 50, key is test_row_0/A:col10/1730983406072/Put/seqid=0 2024-11-07T12:43:26,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742476_1652 (size=7415) 2024-11-07T12:43:26,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-07T12:43:26,647 INFO [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=644 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/fafe66263b3844ba87584d0bc79a51d3 2024-11-07T12:43:26,653 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/3253773184e74eb5aa1230d7e205d914 is 50, key is test_row_0/B:col10/1730983406072/Put/seqid=0 2024-11-07T12:43:26,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742477_1653 (size=7415) 2024-11-07T12:43:26,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-07T12:43:27,057 INFO [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=644 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/3253773184e74eb5aa1230d7e205d914 2024-11-07T12:43:27,063 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/30ef43b65cb849de978989fd752883ed is 50, key is test_row_0/C:col10/1730983406072/Put/seqid=0 2024-11-07T12:43:27,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742478_1654 (size=7415) 2024-11-07T12:43:27,066 INFO [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=644 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/30ef43b65cb849de978989fd752883ed 2024-11-07T12:43:27,069 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/A/fafe66263b3844ba87584d0bc79a51d3 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/fafe66263b3844ba87584d0bc79a51d3 2024-11-07T12:43:27,072 INFO [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/fafe66263b3844ba87584d0bc79a51d3, entries=50, sequenceid=644, filesize=7.2 K 2024-11-07T12:43:27,072 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/B/3253773184e74eb5aa1230d7e205d914 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/3253773184e74eb5aa1230d7e205d914 2024-11-07T12:43:27,075 INFO [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/3253773184e74eb5aa1230d7e205d914, entries=50, sequenceid=644, filesize=7.2 K 2024-11-07T12:43:27,076 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/.tmp/C/30ef43b65cb849de978989fd752883ed as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/30ef43b65cb849de978989fd752883ed 2024-11-07T12:43:27,078 INFO [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/30ef43b65cb849de978989fd752883ed, entries=50, sequenceid=644, filesize=7.2 K 2024-11-07T12:43:27,079 INFO [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 850d939a0dd41b63931f314702af2ad5 in 841ms, sequenceid=644, compaction requested=false 2024-11-07T12:43:27,079 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/ff7ffdbcbb0d48229070491b1c47d133, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/50a0da3c2cda4668a71a2125b9b88859, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/b9852c71c7aa4b3eb458adb00f377b0a, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/76c84375bb6948a8bb0e91d1c15c2efa, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/79bba2e570794c018ba2c753dd14fa06, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/0a46eb4565094f078d600d81f915762e, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/1d6082b99e304fc6b1494a4093179c65, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/7507af023a764d05936b3228797013bf, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/13d09177d3774a3db0af3b089bb8bf15, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/c1e4851b3abb42159c2fb0eb7e7729b7, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/721f5882475c452ba689e2dd4a4736ef, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/f2240a78a5d54d2e86f854ccc8b51c12, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/caaa87115cbd49e8b35b237f1d3fe9a0, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/78beb4d69b6b4ef392c107ba82dd5b66, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/b3486332d47c4c89b240d2cd297207e7, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/cc2f0a28846d47b3b0c1a899601689c0, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/cd34d10eb9ff4ec097055a98c1bc0438, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/a14ab44472144ecfa8b2d466c9612f98, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/1490b50bf5cf4b7d875305722ec3a696, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/024d3bac75f44ff29b76b70530be07ac, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/2c8ffd18227e47b0a36e3282e43fb737, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/0980090c7fbe44cb92a405fd8da5a4f8, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/08475cf79a0c4534a1172925958abf7b, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/2e273050c85a4afea0092a8a8af9f037, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/f9069431666341b58ebf6b6d5dc247cd, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/11ff98c2a23a4dd4a4e266816400f285, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/a6278f4f4c374a21bbca6e2e0d8166fd, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/a90a50348a8344549c1584f2417415a5, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/56c44e94c497468b9dfa8b49e444218d, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/f6ecd18bef2b40eab22598b4730223cd, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/0135c0b0b316427c890f239ac86d2d30, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/913d394be1b24fe5afe6a35eb45eaff4, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/6cb5181a93dd438baa46289b5dd4bcab, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/5fbac044cb6c4a349af35dd77582f0a6, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/a08a587b9e184fdfb06489626165931d, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/7a7e459216a7464ea5d9e7f575fd486a, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/2d2241e109b54637a131dcd1e5775411, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/66cbac03ea85465b9d8cb240691004b0, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/64f0056f35134016b346341eb7f32757, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/5682fc0ac8e6470bbb8f859c8c0fdb3a, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/f60b4f7e611b487aaa6725e12d266e0c] to archive 2024-11-07T12:43:27,080 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T12:43:27,082 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/ff7ffdbcbb0d48229070491b1c47d133 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/ff7ffdbcbb0d48229070491b1c47d133 2024-11-07T12:43:27,083 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/50a0da3c2cda4668a71a2125b9b88859 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/50a0da3c2cda4668a71a2125b9b88859 2024-11-07T12:43:27,084 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/b9852c71c7aa4b3eb458adb00f377b0a to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/b9852c71c7aa4b3eb458adb00f377b0a 2024-11-07T12:43:27,085 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/76c84375bb6948a8bb0e91d1c15c2efa to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/76c84375bb6948a8bb0e91d1c15c2efa 2024-11-07T12:43:27,087 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/79bba2e570794c018ba2c753dd14fa06 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/79bba2e570794c018ba2c753dd14fa06 2024-11-07T12:43:27,088 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/0a46eb4565094f078d600d81f915762e to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/0a46eb4565094f078d600d81f915762e 2024-11-07T12:43:27,089 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/1d6082b99e304fc6b1494a4093179c65 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/1d6082b99e304fc6b1494a4093179c65 2024-11-07T12:43:27,091 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/7507af023a764d05936b3228797013bf to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/7507af023a764d05936b3228797013bf 2024-11-07T12:43:27,092 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/13d09177d3774a3db0af3b089bb8bf15 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/13d09177d3774a3db0af3b089bb8bf15 2024-11-07T12:43:27,093 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/c1e4851b3abb42159c2fb0eb7e7729b7 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/c1e4851b3abb42159c2fb0eb7e7729b7 2024-11-07T12:43:27,094 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/721f5882475c452ba689e2dd4a4736ef to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/721f5882475c452ba689e2dd4a4736ef 2024-11-07T12:43:27,095 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/f2240a78a5d54d2e86f854ccc8b51c12 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/f2240a78a5d54d2e86f854ccc8b51c12 2024-11-07T12:43:27,097 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/caaa87115cbd49e8b35b237f1d3fe9a0 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/caaa87115cbd49e8b35b237f1d3fe9a0 2024-11-07T12:43:27,098 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/78beb4d69b6b4ef392c107ba82dd5b66 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/78beb4d69b6b4ef392c107ba82dd5b66 2024-11-07T12:43:27,099 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/b3486332d47c4c89b240d2cd297207e7 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/b3486332d47c4c89b240d2cd297207e7 2024-11-07T12:43:27,100 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/cc2f0a28846d47b3b0c1a899601689c0 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/cc2f0a28846d47b3b0c1a899601689c0 2024-11-07T12:43:27,101 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/cd34d10eb9ff4ec097055a98c1bc0438 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/cd34d10eb9ff4ec097055a98c1bc0438 2024-11-07T12:43:27,103 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/a14ab44472144ecfa8b2d466c9612f98 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/a14ab44472144ecfa8b2d466c9612f98 2024-11-07T12:43:27,104 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/1490b50bf5cf4b7d875305722ec3a696 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/1490b50bf5cf4b7d875305722ec3a696 2024-11-07T12:43:27,105 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/024d3bac75f44ff29b76b70530be07ac to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/024d3bac75f44ff29b76b70530be07ac 2024-11-07T12:43:27,106 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/2c8ffd18227e47b0a36e3282e43fb737 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/2c8ffd18227e47b0a36e3282e43fb737 2024-11-07T12:43:27,108 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/0980090c7fbe44cb92a405fd8da5a4f8 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/0980090c7fbe44cb92a405fd8da5a4f8 2024-11-07T12:43:27,109 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/08475cf79a0c4534a1172925958abf7b to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/08475cf79a0c4534a1172925958abf7b 2024-11-07T12:43:27,110 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/2e273050c85a4afea0092a8a8af9f037 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/2e273050c85a4afea0092a8a8af9f037 2024-11-07T12:43:27,112 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/f9069431666341b58ebf6b6d5dc247cd to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/f9069431666341b58ebf6b6d5dc247cd 2024-11-07T12:43:27,114 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/11ff98c2a23a4dd4a4e266816400f285 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/11ff98c2a23a4dd4a4e266816400f285 2024-11-07T12:43:27,117 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/a6278f4f4c374a21bbca6e2e0d8166fd to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/a6278f4f4c374a21bbca6e2e0d8166fd 2024-11-07T12:43:27,118 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/a90a50348a8344549c1584f2417415a5 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/a90a50348a8344549c1584f2417415a5 2024-11-07T12:43:27,119 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/56c44e94c497468b9dfa8b49e444218d to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/56c44e94c497468b9dfa8b49e444218d 2024-11-07T12:43:27,119 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/f6ecd18bef2b40eab22598b4730223cd to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/f6ecd18bef2b40eab22598b4730223cd 2024-11-07T12:43:27,121 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/0135c0b0b316427c890f239ac86d2d30 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/0135c0b0b316427c890f239ac86d2d30 2024-11-07T12:43:27,122 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/913d394be1b24fe5afe6a35eb45eaff4 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/913d394be1b24fe5afe6a35eb45eaff4 2024-11-07T12:43:27,124 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/6cb5181a93dd438baa46289b5dd4bcab to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/6cb5181a93dd438baa46289b5dd4bcab 2024-11-07T12:43:27,125 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/5fbac044cb6c4a349af35dd77582f0a6 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/5fbac044cb6c4a349af35dd77582f0a6 2024-11-07T12:43:27,126 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/a08a587b9e184fdfb06489626165931d to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/a08a587b9e184fdfb06489626165931d 2024-11-07T12:43:27,127 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/7a7e459216a7464ea5d9e7f575fd486a to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/7a7e459216a7464ea5d9e7f575fd486a 2024-11-07T12:43:27,128 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/2d2241e109b54637a131dcd1e5775411 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/2d2241e109b54637a131dcd1e5775411 2024-11-07T12:43:27,129 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/66cbac03ea85465b9d8cb240691004b0 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/66cbac03ea85465b9d8cb240691004b0 2024-11-07T12:43:27,130 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/64f0056f35134016b346341eb7f32757 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/64f0056f35134016b346341eb7f32757 2024-11-07T12:43:27,131 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/5682fc0ac8e6470bbb8f859c8c0fdb3a to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/5682fc0ac8e6470bbb8f859c8c0fdb3a 2024-11-07T12:43:27,132 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/f60b4f7e611b487aaa6725e12d266e0c to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/f60b4f7e611b487aaa6725e12d266e0c 2024-11-07T12:43:27,134 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/d04d499c570f4a759b33df3da8044bdc, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/0ff4c794cd93426fae98328b170e285a, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/d88e9bbfc4714ee7be67304353c12338, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/7f21af6315b14191a36cb732e4556b18, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/cdf20744350b42c5a9884fc374a933f4, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/7541e610d3c1416fbac659eee5019d36, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/a63b789119614bd9b6a3f06155e7a592, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/a5b46aaf1bf2488a822ac5b1b1f65154, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/96088dd6609140428ac6f53fca1852c9, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/48010982943b42e4bf3299d7d67a1a2a, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/dad736474d52401298f14efec800557b, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/12b8e69d23e945cea852b93c5efc3d0e, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/d1990c70fd864669a4391d403fd8e3de, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/2a1d7360b89445f98d47e496c9d1fa61, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/d3cf1868acbe44f29774553dcb68064c, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/aa990acda5fe48408886670984010f6a, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/f84ede0489a24be1993917ddb87564ff, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/dd11685f640c452e91fbc824a8fb70e7, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/313fdbb6092a4b7bb8e9a8d21b1d1514, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/22188d97e5f549d7ad395e64100f7223, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/fae4dece023e4e199adfd97e7181d503, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/c748e44bc30a451b9e823b6b91ca4858, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/979e890a72a54ed59831772b0169ff8a, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/ed696a6ee45741f3881ea308257b0416, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/a6c0d7070b544118bb72c89da7a10472, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/45a88ba256374cd398ca0362df121c13, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/9a4df53911d242128baf4f7472a5653f, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/bba81c33d709419f8318529199ab851e, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/c6673fc2cb4949459802fba9bb0a18fc, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/ba592b7fbd5e4a1fbaea0a8d2d336840, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/f048965776a14629a3f0364ffb697c9b, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/e46e742da3cd4cdb9e9a1f9bb0dae2da, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/2c2b1f5daf6447809e7fae99d6be1edd, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/320957b359d942f38310a60943fa6ad6, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/cee0db2678384ef4b62d7f2709a95cf0, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/60749c65e45843d98aa0ea6d4bf65fc7, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/7f7a27ae46da43b5b0a5bbf60e5734ba, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/5f4da1bf5ee143b1ba8bddc4b76db26e, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/068e892d41074dc0b236a325c9be0db3, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/72009bec59f947f3bb234442ed6316c0, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/79c1549d59ca402799e8e6d367fa4610] to archive 2024-11-07T12:43:27,135 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T12:43:27,136 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/d04d499c570f4a759b33df3da8044bdc to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/d04d499c570f4a759b33df3da8044bdc 2024-11-07T12:43:27,137 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/0ff4c794cd93426fae98328b170e285a to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/0ff4c794cd93426fae98328b170e285a 2024-11-07T12:43:27,138 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/d88e9bbfc4714ee7be67304353c12338 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/d88e9bbfc4714ee7be67304353c12338 2024-11-07T12:43:27,139 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/7f21af6315b14191a36cb732e4556b18 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/7f21af6315b14191a36cb732e4556b18 2024-11-07T12:43:27,140 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/cdf20744350b42c5a9884fc374a933f4 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/cdf20744350b42c5a9884fc374a933f4 2024-11-07T12:43:27,141 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/7541e610d3c1416fbac659eee5019d36 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/7541e610d3c1416fbac659eee5019d36 2024-11-07T12:43:27,142 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/a63b789119614bd9b6a3f06155e7a592 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/a63b789119614bd9b6a3f06155e7a592 2024-11-07T12:43:27,142 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/a5b46aaf1bf2488a822ac5b1b1f65154 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/a5b46aaf1bf2488a822ac5b1b1f65154 2024-11-07T12:43:27,143 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/96088dd6609140428ac6f53fca1852c9 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/96088dd6609140428ac6f53fca1852c9 2024-11-07T12:43:27,144 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/48010982943b42e4bf3299d7d67a1a2a to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/48010982943b42e4bf3299d7d67a1a2a 2024-11-07T12:43:27,145 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/dad736474d52401298f14efec800557b to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/dad736474d52401298f14efec800557b 2024-11-07T12:43:27,146 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/12b8e69d23e945cea852b93c5efc3d0e to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/12b8e69d23e945cea852b93c5efc3d0e 2024-11-07T12:43:27,147 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/d1990c70fd864669a4391d403fd8e3de to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/d1990c70fd864669a4391d403fd8e3de 2024-11-07T12:43:27,148 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/2a1d7360b89445f98d47e496c9d1fa61 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/2a1d7360b89445f98d47e496c9d1fa61 2024-11-07T12:43:27,149 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/d3cf1868acbe44f29774553dcb68064c to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/d3cf1868acbe44f29774553dcb68064c 2024-11-07T12:43:27,149 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/aa990acda5fe48408886670984010f6a to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/aa990acda5fe48408886670984010f6a 2024-11-07T12:43:27,150 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/f84ede0489a24be1993917ddb87564ff to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/f84ede0489a24be1993917ddb87564ff 2024-11-07T12:43:27,151 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/dd11685f640c452e91fbc824a8fb70e7 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/dd11685f640c452e91fbc824a8fb70e7 2024-11-07T12:43:27,152 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/313fdbb6092a4b7bb8e9a8d21b1d1514 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/313fdbb6092a4b7bb8e9a8d21b1d1514 2024-11-07T12:43:27,152 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/22188d97e5f549d7ad395e64100f7223 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/22188d97e5f549d7ad395e64100f7223 2024-11-07T12:43:27,153 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/fae4dece023e4e199adfd97e7181d503 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/fae4dece023e4e199adfd97e7181d503 2024-11-07T12:43:27,154 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/c748e44bc30a451b9e823b6b91ca4858 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/c748e44bc30a451b9e823b6b91ca4858 2024-11-07T12:43:27,155 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/979e890a72a54ed59831772b0169ff8a to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/979e890a72a54ed59831772b0169ff8a 2024-11-07T12:43:27,155 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/ed696a6ee45741f3881ea308257b0416 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/ed696a6ee45741f3881ea308257b0416 2024-11-07T12:43:27,156 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/a6c0d7070b544118bb72c89da7a10472 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/a6c0d7070b544118bb72c89da7a10472 2024-11-07T12:43:27,157 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/45a88ba256374cd398ca0362df121c13 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/45a88ba256374cd398ca0362df121c13 2024-11-07T12:43:27,158 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/9a4df53911d242128baf4f7472a5653f to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/9a4df53911d242128baf4f7472a5653f 2024-11-07T12:43:27,158 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/bba81c33d709419f8318529199ab851e to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/bba81c33d709419f8318529199ab851e 2024-11-07T12:43:27,159 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/c6673fc2cb4949459802fba9bb0a18fc to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/c6673fc2cb4949459802fba9bb0a18fc 2024-11-07T12:43:27,160 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/ba592b7fbd5e4a1fbaea0a8d2d336840 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/ba592b7fbd5e4a1fbaea0a8d2d336840 2024-11-07T12:43:27,161 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/f048965776a14629a3f0364ffb697c9b to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/f048965776a14629a3f0364ffb697c9b 2024-11-07T12:43:27,161 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/e46e742da3cd4cdb9e9a1f9bb0dae2da to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/e46e742da3cd4cdb9e9a1f9bb0dae2da 2024-11-07T12:43:27,162 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/2c2b1f5daf6447809e7fae99d6be1edd to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/2c2b1f5daf6447809e7fae99d6be1edd 2024-11-07T12:43:27,163 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/320957b359d942f38310a60943fa6ad6 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/320957b359d942f38310a60943fa6ad6 2024-11-07T12:43:27,164 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/cee0db2678384ef4b62d7f2709a95cf0 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/cee0db2678384ef4b62d7f2709a95cf0 2024-11-07T12:43:27,165 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/60749c65e45843d98aa0ea6d4bf65fc7 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/60749c65e45843d98aa0ea6d4bf65fc7 2024-11-07T12:43:27,166 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/7f7a27ae46da43b5b0a5bbf60e5734ba to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/7f7a27ae46da43b5b0a5bbf60e5734ba 2024-11-07T12:43:27,166 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/5f4da1bf5ee143b1ba8bddc4b76db26e to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/5f4da1bf5ee143b1ba8bddc4b76db26e 2024-11-07T12:43:27,167 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/068e892d41074dc0b236a325c9be0db3 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/068e892d41074dc0b236a325c9be0db3 2024-11-07T12:43:27,168 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/72009bec59f947f3bb234442ed6316c0 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/72009bec59f947f3bb234442ed6316c0 2024-11-07T12:43:27,169 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/79c1549d59ca402799e8e6d367fa4610 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/79c1549d59ca402799e8e6d367fa4610 2024-11-07T12:43:27,170 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/0924f27e2a4947c499be70119cb3e0b6, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/a97d7f6e75fa4e829c0f5fdb479b0db2, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/f24d168431834d9bab9d414bc2006ddc, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/03ee9cce38c04955841ceb4ea0e6abec, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/2c7b7c5118e24ff7b1d99b6cd0d2a0c5, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/27ca2d6c113e436b91e8e333eb527743, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/ef5e053528fc4de794b3f37b3c697aa2, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/5a29b9204dc94f8aa6e94116512cd122, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/5df70c77ac1e499c87de84f6dd8652e4, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/841899e585b84216ae16585f18802c3f, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/d722000c169245bc987716f59b6c21b5, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/2b6e6844916c46ceb0b2e690f073f2d0, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/96adca44c84f49a390335a6d6c4eb5c7, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/51bca35d4f0f4b548ee81f7971358962, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/b7edaefc324947deade9146a2a2f153d, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/e14e8d11df9a40a99bb4253f9d234108, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/d4b3de077bcc472ea350170dccbf424d, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/f779f664483745c197cca3200851c245, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/7d7f59f69c6344b8b2334b608b9b05e1, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/103117f806a6451eb7e31d259e6bbcc3, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/c7f9c8fe7b194819871450c0d2bcaeef, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/7f59551171924483b6da82489316136d, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/7667a5ba169c4b92bccbe628870dbab1, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/a21c8917ccb64939bf3c5676ee522fe4, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/282c586e976e4fe2ab919dad59c99ec6, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/f159b9f8ab6b488f92d41bfb6cff228a, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/7ed61756dbc04853b62df506914a6be9, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/09d14d162e134ac0990b1d8c364ddf19, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/0626691b740e49b69a54da0b8fe7eeb7, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/b0310ca93d1f42fe9f4283299a76ffba, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/078ee72674144e07896f6c9773c9b62e, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/13692a90aac24bb1a276669a848dee7c, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/38de3ee681e24ddeac6959751bd7b842, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/09a3391ebbfe4fe9bedb237326c36370, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/7ee0db856c7b4cacae2c7e875e6678f5, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/2578e9695153461db5ed80b84c4d5a24, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/c8b3e751295b423793bf15f6b2eccc0a, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/2880b70564cd4f5e8dc436ad40a34a36, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/f94e9ed0d5a745df91c61521d90dc83b] to archive 2024-11-07T12:43:27,171 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T12:43:27,172 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/0924f27e2a4947c499be70119cb3e0b6 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/0924f27e2a4947c499be70119cb3e0b6 2024-11-07T12:43:27,172 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/a97d7f6e75fa4e829c0f5fdb479b0db2 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/a97d7f6e75fa4e829c0f5fdb479b0db2 2024-11-07T12:43:27,173 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/f24d168431834d9bab9d414bc2006ddc to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/f24d168431834d9bab9d414bc2006ddc 2024-11-07T12:43:27,174 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/03ee9cce38c04955841ceb4ea0e6abec to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/03ee9cce38c04955841ceb4ea0e6abec 2024-11-07T12:43:27,174 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/2c7b7c5118e24ff7b1d99b6cd0d2a0c5 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/2c7b7c5118e24ff7b1d99b6cd0d2a0c5 2024-11-07T12:43:27,175 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/27ca2d6c113e436b91e8e333eb527743 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/27ca2d6c113e436b91e8e333eb527743 2024-11-07T12:43:27,176 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/ef5e053528fc4de794b3f37b3c697aa2 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/ef5e053528fc4de794b3f37b3c697aa2 2024-11-07T12:43:27,177 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/5a29b9204dc94f8aa6e94116512cd122 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/5a29b9204dc94f8aa6e94116512cd122 2024-11-07T12:43:27,177 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/5df70c77ac1e499c87de84f6dd8652e4 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/5df70c77ac1e499c87de84f6dd8652e4 2024-11-07T12:43:27,178 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/841899e585b84216ae16585f18802c3f to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/841899e585b84216ae16585f18802c3f 2024-11-07T12:43:27,179 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/d722000c169245bc987716f59b6c21b5 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/d722000c169245bc987716f59b6c21b5 2024-11-07T12:43:27,180 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/2b6e6844916c46ceb0b2e690f073f2d0 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/2b6e6844916c46ceb0b2e690f073f2d0 2024-11-07T12:43:27,181 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/96adca44c84f49a390335a6d6c4eb5c7 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/96adca44c84f49a390335a6d6c4eb5c7 2024-11-07T12:43:27,181 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/51bca35d4f0f4b548ee81f7971358962 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/51bca35d4f0f4b548ee81f7971358962 2024-11-07T12:43:27,182 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/b7edaefc324947deade9146a2a2f153d to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/b7edaefc324947deade9146a2a2f153d 2024-11-07T12:43:27,183 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/e14e8d11df9a40a99bb4253f9d234108 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/e14e8d11df9a40a99bb4253f9d234108 2024-11-07T12:43:27,184 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/d4b3de077bcc472ea350170dccbf424d to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/d4b3de077bcc472ea350170dccbf424d 2024-11-07T12:43:27,185 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/f779f664483745c197cca3200851c245 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/f779f664483745c197cca3200851c245 2024-11-07T12:43:27,185 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/7d7f59f69c6344b8b2334b608b9b05e1 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/7d7f59f69c6344b8b2334b608b9b05e1 2024-11-07T12:43:27,186 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/103117f806a6451eb7e31d259e6bbcc3 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/103117f806a6451eb7e31d259e6bbcc3 2024-11-07T12:43:27,187 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/c7f9c8fe7b194819871450c0d2bcaeef to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/c7f9c8fe7b194819871450c0d2bcaeef 2024-11-07T12:43:27,188 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/7f59551171924483b6da82489316136d to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/7f59551171924483b6da82489316136d 2024-11-07T12:43:27,188 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/7667a5ba169c4b92bccbe628870dbab1 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/7667a5ba169c4b92bccbe628870dbab1 2024-11-07T12:43:27,189 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/a21c8917ccb64939bf3c5676ee522fe4 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/a21c8917ccb64939bf3c5676ee522fe4 2024-11-07T12:43:27,190 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/282c586e976e4fe2ab919dad59c99ec6 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/282c586e976e4fe2ab919dad59c99ec6 2024-11-07T12:43:27,190 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/f159b9f8ab6b488f92d41bfb6cff228a to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/f159b9f8ab6b488f92d41bfb6cff228a 2024-11-07T12:43:27,191 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/7ed61756dbc04853b62df506914a6be9 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/7ed61756dbc04853b62df506914a6be9 2024-11-07T12:43:27,192 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/09d14d162e134ac0990b1d8c364ddf19 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/09d14d162e134ac0990b1d8c364ddf19 2024-11-07T12:43:27,193 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/0626691b740e49b69a54da0b8fe7eeb7 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/0626691b740e49b69a54da0b8fe7eeb7 2024-11-07T12:43:27,193 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/b0310ca93d1f42fe9f4283299a76ffba to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/b0310ca93d1f42fe9f4283299a76ffba 2024-11-07T12:43:27,194 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/078ee72674144e07896f6c9773c9b62e to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/078ee72674144e07896f6c9773c9b62e 2024-11-07T12:43:27,195 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/13692a90aac24bb1a276669a848dee7c to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/13692a90aac24bb1a276669a848dee7c 2024-11-07T12:43:27,196 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/38de3ee681e24ddeac6959751bd7b842 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/38de3ee681e24ddeac6959751bd7b842 2024-11-07T12:43:27,197 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/09a3391ebbfe4fe9bedb237326c36370 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/09a3391ebbfe4fe9bedb237326c36370 2024-11-07T12:43:27,197 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/7ee0db856c7b4cacae2c7e875e6678f5 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/7ee0db856c7b4cacae2c7e875e6678f5 2024-11-07T12:43:27,198 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/2578e9695153461db5ed80b84c4d5a24 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/2578e9695153461db5ed80b84c4d5a24 2024-11-07T12:43:27,199 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/c8b3e751295b423793bf15f6b2eccc0a to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/c8b3e751295b423793bf15f6b2eccc0a 2024-11-07T12:43:27,200 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/2880b70564cd4f5e8dc436ad40a34a36 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/2880b70564cd4f5e8dc436ad40a34a36 2024-11-07T12:43:27,201 DEBUG [StoreCloser-TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/f94e9ed0d5a745df91c61521d90dc83b to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/f94e9ed0d5a745df91c61521d90dc83b 2024-11-07T12:43:27,204 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/recovered.edits/647.seqid, newMaxSeqId=647, maxSeqId=1 2024-11-07T12:43:27,205 INFO [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. 2024-11-07T12:43:27,205 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1676): Region close journal for 850d939a0dd41b63931f314702af2ad5: Waiting for close lock at 1730983406237Running coprocessor pre-close hooks at 1730983406237Disabling compacts and flushes for region at 1730983406237Disabling writes for close at 1730983406238 (+1 ms)Obtaining lock to block concurrent updates at 1730983406238Preparing flush snapshotting stores in 850d939a0dd41b63931f314702af2ad5 at 1730983406238Finished memstore snapshotting TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=41520, getOffHeapSize=0, getCellsCount=300 at 1730983406238Flushing stores of TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5. at 1730983406239 (+1 ms)Flushing 850d939a0dd41b63931f314702af2ad5/A: creating writer at 1730983406239Flushing 850d939a0dd41b63931f314702af2ad5/A: appending metadata at 1730983406241 (+2 ms)Flushing 850d939a0dd41b63931f314702af2ad5/A: closing flushed file at 1730983406241Flushing 850d939a0dd41b63931f314702af2ad5/B: creating writer at 1730983406650 (+409 ms)Flushing 850d939a0dd41b63931f314702af2ad5/B: appending metadata at 1730983406652 (+2 ms)Flushing 850d939a0dd41b63931f314702af2ad5/B: closing flushed file at 1730983406653 (+1 ms)Flushing 850d939a0dd41b63931f314702af2ad5/C: creating writer at 1730983407060 (+407 ms)Flushing 850d939a0dd41b63931f314702af2ad5/C: appending metadata at 1730983407062 (+2 ms)Flushing 850d939a0dd41b63931f314702af2ad5/C: closing flushed file at 1730983407062Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@839a705: reopening flushed file at 1730983407068 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3f9248d0: reopening flushed file at 1730983407072 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@260fd0fb: reopening flushed file at 1730983407075 (+3 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 850d939a0dd41b63931f314702af2ad5 in 841ms, sequenceid=644, compaction requested=false at 1730983407079 (+4 ms)Writing region close event to WAL at 1730983407202 (+123 ms)Running coprocessor post-close hooks at 1730983407205 (+3 ms)Closed at 1730983407205 2024-11-07T12:43:27,207 INFO [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(157): Closed 850d939a0dd41b63931f314702af2ad5 2024-11-07T12:43:27,208 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=850d939a0dd41b63931f314702af2ad5, regionState=CLOSED 2024-11-07T12:43:27,209 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=148, ppid=147, state=RUNNABLE, hasLock=false; CloseRegionProcedure 850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 because future has completed 2024-11-07T12:43:27,211 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-11-07T12:43:27,211 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; CloseRegionProcedure 850d939a0dd41b63931f314702af2ad5, server=db9ad1cb6cf9,34293,1730983260217 in 1.1250 sec 2024-11-07T12:43:27,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-07T12:43:27,213 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-07T12:43:27,213 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=850d939a0dd41b63931f314702af2ad5, UNASSIGN in 1.1290 sec 2024-11-07T12:43:27,215 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-07T12:43:27,215 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.1330 sec 2024-11-07T12:43:27,216 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730983407216"}]},"ts":"1730983407216"} 2024-11-07T12:43:27,217 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-07T12:43:27,217 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-07T12:43:27,219 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.1430 sec 2024-11-07T12:43:27,837 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-07T12:43:28,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-07T12:43:28,223 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:43:28,223 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-07T12:43:28,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] procedure2.ProcedureExecutor(1139): Stored pid=149, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:43:28,225 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=149, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:43:28,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-11-07T12:43:28,226 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=149, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:43:28,229 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5 2024-11-07T12:43:28,232 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A, FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B, FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C, FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/recovered.edits] 2024-11-07T12:43:28,235 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/a32b8c4d019e43cbaf7a0a803ad5eac1 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/a32b8c4d019e43cbaf7a0a803ad5eac1 2024-11-07T12:43:28,236 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/fafe66263b3844ba87584d0bc79a51d3 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/A/fafe66263b3844ba87584d0bc79a51d3 2024-11-07T12:43:28,238 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/3253773184e74eb5aa1230d7e205d914 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/3253773184e74eb5aa1230d7e205d914 2024-11-07T12:43:28,240 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/b6d333ee3895438bb5dae0a1d04a29c3 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/B/b6d333ee3895438bb5dae0a1d04a29c3 2024-11-07T12:43:28,242 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/30ef43b65cb849de978989fd752883ed to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/30ef43b65cb849de978989fd752883ed 2024-11-07T12:43:28,243 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/6973784b88934758b0506601aac4c118 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/C/6973784b88934758b0506601aac4c118 2024-11-07T12:43:28,246 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/recovered.edits/647.seqid to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5/recovered.edits/647.seqid 2024-11-07T12:43:28,247 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/850d939a0dd41b63931f314702af2ad5 2024-11-07T12:43:28,247 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-07T12:43:28,250 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=149, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:43:28,252 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-07T12:43:28,255 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-07T12:43:28,256 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=149, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:43:28,256 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-07T12:43:28,256 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1730983408256"}]},"ts":"9223372036854775807"} 2024-11-07T12:43:28,258 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-07T12:43:28,258 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 850d939a0dd41b63931f314702af2ad5, NAME => 'TestAcidGuarantees,,1730983379395.850d939a0dd41b63931f314702af2ad5.', STARTKEY => '', ENDKEY => ''}] 2024-11-07T12:43:28,258 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-07T12:43:28,259 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1730983408258"}]},"ts":"9223372036854775807"} 2024-11-07T12:43:28,260 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-07T12:43:28,261 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=149, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:43:28,262 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 38 msec 2024-11-07T12:43:28,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-11-07T12:43:28,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-07T12:43:28,332 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:43:28,342 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=249 (was 250), OpenFileDescriptor=453 (was 463), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=861 (was 985), ProcessCount=11 (was 11), AvailableMemoryMB=2505 (was 2925) 2024-11-07T12:43:28,351 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=249, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=861, ProcessCount=11, AvailableMemoryMB=2504 2024-11-07T12:43:28,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-07T12:43:28,353 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T12:43:28,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-07T12:43:28,355 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=150, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T12:43:28,355 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 150 2024-11-07T12:43:28,355 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:28,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-07T12:43:28,356 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=150, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T12:43:28,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742479_1655 (size=1061) 2024-11-07T12:43:28,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-07T12:43:28,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-07T12:43:28,762 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4 2024-11-07T12:43:28,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742480_1656 (size=53) 2024-11-07T12:43:28,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-07T12:43:29,167 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:43:29,167 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 37eda6b0dad56caa0fa6c1ac8208d5b2, disabling compactions & flushes 2024-11-07T12:43:29,167 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:29,167 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:29,167 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. after waiting 0 ms 2024-11-07T12:43:29,167 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:29,167 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:29,167 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 37eda6b0dad56caa0fa6c1ac8208d5b2: Waiting for close lock at 1730983409167Disabling compacts and flushes for region at 1730983409167Disabling writes for close at 1730983409167Writing region close event to WAL at 1730983409167Closed at 1730983409167 2024-11-07T12:43:29,169 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=150, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T12:43:29,169 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1730983409169"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730983409169"}]},"ts":"1730983409169"} 2024-11-07T12:43:29,170 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-07T12:43:29,171 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=150, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T12:43:29,171 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730983409171"}]},"ts":"1730983409171"} 2024-11-07T12:43:29,172 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-07T12:43:29,173 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=37eda6b0dad56caa0fa6c1ac8208d5b2, ASSIGN}] 2024-11-07T12:43:29,173 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=37eda6b0dad56caa0fa6c1ac8208d5b2, ASSIGN 2024-11-07T12:43:29,174 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=37eda6b0dad56caa0fa6c1ac8208d5b2, ASSIGN; state=OFFLINE, location=db9ad1cb6cf9,34293,1730983260217; forceNewPlan=false, retain=false 2024-11-07T12:43:29,325 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=37eda6b0dad56caa0fa6c1ac8208d5b2, regionState=OPENING, regionLocation=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:29,327 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=37eda6b0dad56caa0fa6c1ac8208d5b2, ASSIGN because future has completed 2024-11-07T12:43:29,327 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=152, ppid=151, state=RUNNABLE, hasLock=false; OpenRegionProcedure 37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217}] 2024-11-07T12:43:29,482 INFO [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:29,482 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(7752): Opening region: {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} 2024-11-07T12:43:29,483 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:29,483 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:43:29,483 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(7794): checking encryption for 37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:29,483 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(7797): checking classloading for 37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:29,489 INFO [StoreOpener-37eda6b0dad56caa0fa6c1ac8208d5b2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:29,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-07T12:43:29,493 INFO [StoreOpener-37eda6b0dad56caa0fa6c1ac8208d5b2-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-07T12:43:29,494 INFO [StoreOpener-37eda6b0dad56caa0fa6c1ac8208d5b2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 37eda6b0dad56caa0fa6c1ac8208d5b2 columnFamilyName A 2024-11-07T12:43:29,494 DEBUG [StoreOpener-37eda6b0dad56caa0fa6c1ac8208d5b2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:29,497 INFO [StoreOpener-37eda6b0dad56caa0fa6c1ac8208d5b2-1 {}] regionserver.HStore(327): Store=37eda6b0dad56caa0fa6c1ac8208d5b2/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:43:29,497 INFO [StoreOpener-37eda6b0dad56caa0fa6c1ac8208d5b2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:29,502 INFO [StoreOpener-37eda6b0dad56caa0fa6c1ac8208d5b2-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-07T12:43:29,502 INFO [StoreOpener-37eda6b0dad56caa0fa6c1ac8208d5b2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 37eda6b0dad56caa0fa6c1ac8208d5b2 columnFamilyName B 2024-11-07T12:43:29,502 DEBUG [StoreOpener-37eda6b0dad56caa0fa6c1ac8208d5b2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:29,503 INFO [StoreOpener-37eda6b0dad56caa0fa6c1ac8208d5b2-1 {}] regionserver.HStore(327): Store=37eda6b0dad56caa0fa6c1ac8208d5b2/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:43:29,503 INFO [StoreOpener-37eda6b0dad56caa0fa6c1ac8208d5b2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:29,505 INFO [StoreOpener-37eda6b0dad56caa0fa6c1ac8208d5b2-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-07T12:43:29,505 INFO [StoreOpener-37eda6b0dad56caa0fa6c1ac8208d5b2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 37eda6b0dad56caa0fa6c1ac8208d5b2 columnFamilyName C 2024-11-07T12:43:29,505 DEBUG [StoreOpener-37eda6b0dad56caa0fa6c1ac8208d5b2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:29,506 INFO [StoreOpener-37eda6b0dad56caa0fa6c1ac8208d5b2-1 {}] regionserver.HStore(327): Store=37eda6b0dad56caa0fa6c1ac8208d5b2/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:43:29,506 INFO [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:29,506 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(1038): replaying wal for 37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:29,507 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:29,507 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:29,508 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(1048): stopping wal replay for 37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:29,508 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(1060): Cleaning up temporary data for 37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:29,512 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-07T12:43:29,514 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(1093): writing seq id for 37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:29,519 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T12:43:29,519 INFO [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(1114): Opened 37eda6b0dad56caa0fa6c1ac8208d5b2; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61580195, jitterRate=-0.08238358795642853}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-07T12:43:29,519 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:29,520 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(1006): Region open journal for 37eda6b0dad56caa0fa6c1ac8208d5b2: Running coprocessor pre-open hook at 1730983409483Writing region info on filesystem at 1730983409483Initializing all the Stores at 1730983409489 (+6 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730983409489Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730983409489Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730983409489Cleaning up temporary data from old regions at 1730983409508 (+19 ms)Running coprocessor post-open hooks at 1730983409519 (+11 ms)Region opened successfully at 1730983409520 (+1 ms) 2024-11-07T12:43:29,521 INFO [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., pid=152, masterSystemTime=1730983409479 2024-11-07T12:43:29,525 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:29,525 INFO [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:29,529 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=37eda6b0dad56caa0fa6c1ac8208d5b2, regionState=OPEN, openSeqNum=2, regionLocation=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:29,531 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=152, ppid=151, state=RUNNABLE, hasLock=false; OpenRegionProcedure 37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 because future has completed 2024-11-07T12:43:29,534 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=152, resume processing ppid=151 2024-11-07T12:43:29,534 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, ppid=151, state=SUCCESS, hasLock=false; OpenRegionProcedure 37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 in 205 msec 2024-11-07T12:43:29,536 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-07T12:43:29,536 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=37eda6b0dad56caa0fa6c1ac8208d5b2, ASSIGN in 361 msec 2024-11-07T12:43:29,537 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=150, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T12:43:29,537 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730983409537"}]},"ts":"1730983409537"} 2024-11-07T12:43:29,539 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-07T12:43:29,540 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=150, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T12:43:29,541 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1870 sec 2024-11-07T12:43:30,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-07T12:43:30,512 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:43:30,513 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@185b2823, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:43:30,513 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request db9ad1cb6cf9,38619,-1 for getting cluster id 2024-11-07T12:43:30,513 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:43:30,516 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e542096-4b0e-4b01-9952-662501fc0f65' 2024-11-07T12:43:30,517 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:43:30,517 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e542096-4b0e-4b01-9952-662501fc0f65" 2024-11-07T12:43:30,517 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38f2679e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:43:30,517 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [db9ad1cb6cf9,38619,-1] 2024-11-07T12:43:30,517 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:43:30,518 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:43:30,518 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43668, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:43:30,519 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9e0048d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:43:30,520 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:43:30,521 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=-1] 2024-11-07T12:43:30,521 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:43:30,522 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47918, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:43:30,523 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-07T12:43:30,524 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is db9ad1cb6cf9,38619,1730983259119 2024-11-07T12:43:30,524 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@12b4b19f 2024-11-07T12:43:30,525 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-07T12:43:30,525 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43676, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-07T12:43:30,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-07T12:43:30,527 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T12:43:30,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] procedure2.ProcedureExecutor(1139): Stored pid=153, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-07T12:43:30,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742481_1657 (size=1097) 2024-11-07T12:43:30,550 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-07T12:43:30,550 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-07T12:43:30,553 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=154, ppid=153, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-07T12:43:30,556 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=37eda6b0dad56caa0fa6c1ac8208d5b2, REOPEN/MOVE}] 2024-11-07T12:43:30,557 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=155, ppid=154, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=37eda6b0dad56caa0fa6c1ac8208d5b2, REOPEN/MOVE 2024-11-07T12:43:30,558 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=155 updating hbase:meta row=37eda6b0dad56caa0fa6c1ac8208d5b2, regionState=CLOSING, regionLocation=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:30,560 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=155, ppid=154, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=37eda6b0dad56caa0fa6c1ac8208d5b2, REOPEN/MOVE because future has completed 2024-11-07T12:43:30,560 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-07T12:43:30,560 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=156, ppid=155, state=RUNNABLE, hasLock=false; CloseRegionProcedure 37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217}] 2024-11-07T12:43:30,712 INFO [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] handler.UnassignRegionHandler(122): Close 37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:30,712 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-07T12:43:30,712 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(1722): Closing 37eda6b0dad56caa0fa6c1ac8208d5b2, disabling compactions & flushes 2024-11-07T12:43:30,712 INFO [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:30,712 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:30,712 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. after waiting 0 ms 2024-11-07T12:43:30,712 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:30,715 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-07T12:43:30,716 INFO [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:30,716 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(1676): Region close journal for 37eda6b0dad56caa0fa6c1ac8208d5b2: Waiting for close lock at 1730983410712Running coprocessor pre-close hooks at 1730983410712Disabling compacts and flushes for region at 1730983410712Disabling writes for close at 1730983410712Writing region close event to WAL at 1730983410712Running coprocessor post-close hooks at 1730983410716 (+4 ms)Closed at 1730983410716 2024-11-07T12:43:30,716 WARN [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegionServer(3299): Not adding moved region record: 37eda6b0dad56caa0fa6c1ac8208d5b2 to self. 2024-11-07T12:43:30,718 INFO [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] handler.UnassignRegionHandler(157): Closed 37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:30,718 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=155 updating hbase:meta row=37eda6b0dad56caa0fa6c1ac8208d5b2, regionState=CLOSED 2024-11-07T12:43:30,720 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=156, ppid=155, state=RUNNABLE, hasLock=false; CloseRegionProcedure 37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 because future has completed 2024-11-07T12:43:30,722 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=156, resume processing ppid=155 2024-11-07T12:43:30,722 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, ppid=155, state=SUCCESS, hasLock=false; CloseRegionProcedure 37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 in 160 msec 2024-11-07T12:43:30,723 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=155, ppid=154, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=37eda6b0dad56caa0fa6c1ac8208d5b2, REOPEN/MOVE; state=CLOSED, location=db9ad1cb6cf9,34293,1730983260217; forceNewPlan=false, retain=true 2024-11-07T12:43:30,873 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=155 updating hbase:meta row=37eda6b0dad56caa0fa6c1ac8208d5b2, regionState=OPENING, regionLocation=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:30,875 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=155, ppid=154, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=37eda6b0dad56caa0fa6c1ac8208d5b2, REOPEN/MOVE because future has completed 2024-11-07T12:43:30,875 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=155, state=RUNNABLE, hasLock=false; OpenRegionProcedure 37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217}] 2024-11-07T12:43:31,029 INFO [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:31,029 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(7752): Opening region: {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} 2024-11-07T12:43:31,030 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:31,030 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:43:31,030 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(7794): checking encryption for 37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:31,030 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(7797): checking classloading for 37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:31,031 INFO [StoreOpener-37eda6b0dad56caa0fa6c1ac8208d5b2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:31,031 INFO [StoreOpener-37eda6b0dad56caa0fa6c1ac8208d5b2-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-07T12:43:31,032 INFO [StoreOpener-37eda6b0dad56caa0fa6c1ac8208d5b2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 37eda6b0dad56caa0fa6c1ac8208d5b2 columnFamilyName A 2024-11-07T12:43:31,032 DEBUG [StoreOpener-37eda6b0dad56caa0fa6c1ac8208d5b2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:31,033 INFO [StoreOpener-37eda6b0dad56caa0fa6c1ac8208d5b2-1 {}] regionserver.HStore(327): Store=37eda6b0dad56caa0fa6c1ac8208d5b2/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:43:31,033 INFO [StoreOpener-37eda6b0dad56caa0fa6c1ac8208d5b2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:31,033 INFO [StoreOpener-37eda6b0dad56caa0fa6c1ac8208d5b2-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-07T12:43:31,034 INFO [StoreOpener-37eda6b0dad56caa0fa6c1ac8208d5b2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 37eda6b0dad56caa0fa6c1ac8208d5b2 columnFamilyName B 2024-11-07T12:43:31,034 DEBUG [StoreOpener-37eda6b0dad56caa0fa6c1ac8208d5b2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:31,034 INFO [StoreOpener-37eda6b0dad56caa0fa6c1ac8208d5b2-1 {}] regionserver.HStore(327): Store=37eda6b0dad56caa0fa6c1ac8208d5b2/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:43:31,034 INFO [StoreOpener-37eda6b0dad56caa0fa6c1ac8208d5b2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:31,034 INFO [StoreOpener-37eda6b0dad56caa0fa6c1ac8208d5b2-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-07T12:43:31,034 INFO [StoreOpener-37eda6b0dad56caa0fa6c1ac8208d5b2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 37eda6b0dad56caa0fa6c1ac8208d5b2 columnFamilyName C 2024-11-07T12:43:31,035 DEBUG [StoreOpener-37eda6b0dad56caa0fa6c1ac8208d5b2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:31,035 INFO [StoreOpener-37eda6b0dad56caa0fa6c1ac8208d5b2-1 {}] regionserver.HStore(327): Store=37eda6b0dad56caa0fa6c1ac8208d5b2/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:43:31,035 INFO [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:31,035 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(1038): replaying wal for 37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:31,035 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:31,036 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:31,037 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(1048): stopping wal replay for 37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:31,037 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(1060): Cleaning up temporary data for 37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:31,037 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-07T12:43:31,038 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(1093): writing seq id for 37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:31,038 INFO [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(1114): Opened 37eda6b0dad56caa0fa6c1ac8208d5b2; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59714122, jitterRate=-0.1101902425289154}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-07T12:43:31,039 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:31,039 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(1006): Region open journal for 37eda6b0dad56caa0fa6c1ac8208d5b2: Running coprocessor pre-open hook at 1730983411030Writing region info on filesystem at 1730983411030Initializing all the Stores at 1730983411030Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730983411030Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730983411031 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730983411031Cleaning up temporary data from old regions at 1730983411037 (+6 ms)Running coprocessor post-open hooks at 1730983411039 (+2 ms)Region opened successfully at 1730983411039 2024-11-07T12:43:31,040 INFO [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., pid=157, masterSystemTime=1730983411027 2024-11-07T12:43:31,041 DEBUG [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:31,041 INFO [RS_OPEN_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:31,042 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=155 updating hbase:meta row=37eda6b0dad56caa0fa6c1ac8208d5b2, regionState=OPEN, openSeqNum=5, regionLocation=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:31,043 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=157, ppid=155, state=RUNNABLE, hasLock=false; OpenRegionProcedure 37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 because future has completed 2024-11-07T12:43:31,045 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=155 2024-11-07T12:43:31,045 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=155, state=SUCCESS, hasLock=false; OpenRegionProcedure 37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 in 169 msec 2024-11-07T12:43:31,047 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-07T12:43:31,047 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=37eda6b0dad56caa0fa6c1ac8208d5b2, REOPEN/MOVE in 489 msec 2024-11-07T12:43:31,049 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=154, resume processing ppid=153 2024-11-07T12:43:31,049 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, ppid=153, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 494 msec 2024-11-07T12:43:31,050 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 521 msec 2024-11-07T12:43:31,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=153 2024-11-07T12:43:31,051 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:43:31,052 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d29c9e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:43:31,052 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request db9ad1cb6cf9,38619,-1 for getting cluster id 2024-11-07T12:43:31,052 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:43:31,053 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e542096-4b0e-4b01-9952-662501fc0f65' 2024-11-07T12:43:31,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:43:31,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e542096-4b0e-4b01-9952-662501fc0f65" 2024-11-07T12:43:31,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@286e4fbf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:43:31,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [db9ad1cb6cf9,38619,-1] 2024-11-07T12:43:31,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:43:31,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:43:31,054 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43690, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:43:31,054 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4585b274, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:43:31,055 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1572f078, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:43:31,055 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request db9ad1cb6cf9,38619,-1 for getting cluster id 2024-11-07T12:43:31,055 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:43:31,056 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e542096-4b0e-4b01-9952-662501fc0f65' 2024-11-07T12:43:31,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:43:31,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e542096-4b0e-4b01-9952-662501fc0f65" 2024-11-07T12:43:31,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b255996, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:43:31,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [db9ad1cb6cf9,38619,-1] 2024-11-07T12:43:31,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:43:31,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:43:31,057 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43708, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:43:31,057 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d0ab602, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:43:31,058 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23448b6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:43:31,058 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request db9ad1cb6cf9,38619,-1 for getting cluster id 2024-11-07T12:43:31,058 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:43:31,058 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e542096-4b0e-4b01-9952-662501fc0f65' 2024-11-07T12:43:31,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:43:31,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e542096-4b0e-4b01-9952-662501fc0f65" 2024-11-07T12:43:31,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23ec0eb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:43:31,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [db9ad1cb6cf9,38619,-1] 2024-11-07T12:43:31,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:43:31,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:43:31,059 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43726, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:43:31,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d49fc76, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:43:31,061 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d7db06e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:43:31,061 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request db9ad1cb6cf9,38619,-1 for getting cluster id 2024-11-07T12:43:31,061 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:43:31,062 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e542096-4b0e-4b01-9952-662501fc0f65' 2024-11-07T12:43:31,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:43:31,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e542096-4b0e-4b01-9952-662501fc0f65" 2024-11-07T12:43:31,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b0af10c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:43:31,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [db9ad1cb6cf9,38619,-1] 2024-11-07T12:43:31,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:43:31,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:43:31,063 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43754, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:43:31,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b737292, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:43:31,064 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@763ca8cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:43:31,064 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request db9ad1cb6cf9,38619,-1 for getting cluster id 2024-11-07T12:43:31,064 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:43:31,064 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e542096-4b0e-4b01-9952-662501fc0f65' 2024-11-07T12:43:31,064 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:43:31,065 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e542096-4b0e-4b01-9952-662501fc0f65" 2024-11-07T12:43:31,065 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a76f16, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:43:31,065 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [db9ad1cb6cf9,38619,-1] 2024-11-07T12:43:31,065 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:43:31,065 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:43:31,065 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43766, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:43:31,066 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b1962ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:43:31,066 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5451f0cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:43:31,066 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request db9ad1cb6cf9,38619,-1 for getting cluster id 2024-11-07T12:43:31,067 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:43:31,067 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e542096-4b0e-4b01-9952-662501fc0f65' 2024-11-07T12:43:31,067 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:43:31,067 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e542096-4b0e-4b01-9952-662501fc0f65" 2024-11-07T12:43:31,067 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@583d3a8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:43:31,067 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [db9ad1cb6cf9,38619,-1] 2024-11-07T12:43:31,067 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:43:31,068 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:43:31,068 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43784, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:43:31,069 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30a2f0ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:43:31,069 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b0e0fb5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:43:31,069 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request db9ad1cb6cf9,38619,-1 for getting cluster id 2024-11-07T12:43:31,069 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:43:31,070 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e542096-4b0e-4b01-9952-662501fc0f65' 2024-11-07T12:43:31,070 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:43:31,070 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e542096-4b0e-4b01-9952-662501fc0f65" 2024-11-07T12:43:31,070 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ba2556d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:43:31,070 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [db9ad1cb6cf9,38619,-1] 2024-11-07T12:43:31,070 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:43:31,071 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:43:31,071 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43802, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:43:31,072 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19c17205, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:43:31,072 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cbb8a41, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:43:31,072 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request db9ad1cb6cf9,38619,-1 for getting cluster id 2024-11-07T12:43:31,072 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:43:31,073 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e542096-4b0e-4b01-9952-662501fc0f65' 2024-11-07T12:43:31,073 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:43:31,073 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e542096-4b0e-4b01-9952-662501fc0f65" 2024-11-07T12:43:31,073 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@423ea92f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:43:31,073 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [db9ad1cb6cf9,38619,-1] 2024-11-07T12:43:31,073 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:43:31,073 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:43:31,074 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43828, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:43:31,074 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b870b37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:43:31,075 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31a58f58, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:43:31,075 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request db9ad1cb6cf9,38619,-1 for getting cluster id 2024-11-07T12:43:31,075 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:43:31,076 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e542096-4b0e-4b01-9952-662501fc0f65' 2024-11-07T12:43:31,076 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:43:31,076 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e542096-4b0e-4b01-9952-662501fc0f65" 2024-11-07T12:43:31,076 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70dfd0af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:43:31,076 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [db9ad1cb6cf9,38619,-1] 2024-11-07T12:43:31,076 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:43:31,076 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:43:31,077 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43842, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:43:31,077 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6824319a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:43:31,078 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e766e1e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:43:31,078 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request db9ad1cb6cf9,38619,-1 for getting cluster id 2024-11-07T12:43:31,078 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:43:31,079 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e542096-4b0e-4b01-9952-662501fc0f65' 2024-11-07T12:43:31,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:43:31,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e542096-4b0e-4b01-9952-662501fc0f65" 2024-11-07T12:43:31,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e648789, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:43:31,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [db9ad1cb6cf9,38619,-1] 2024-11-07T12:43:31,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:43:31,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:43:31,080 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43860, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:43:31,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c54db0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:43:31,081 DEBUG [Thread-2889 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:43:31,081 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=-1] 2024-11-07T12:43:31,082 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:43:31,082 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47924, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:43:31,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5] 2024-11-07T12:43:31,085 DEBUG [Thread-2891 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:43:31,085 DEBUG [Thread-2882 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:43:31,085 DEBUG [Thread-2880 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:43:31,085 DEBUG [Thread-2876 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:43:31,085 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=-1] 2024-11-07T12:43:31,086 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:43:31,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=-1] 2024-11-07T12:43:31,086 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=-1] 2024-11-07T12:43:31,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:43:31,086 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:43:31,087 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47926, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:43:31,087 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47938, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:43:31,087 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47946, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:43:31,088 DEBUG [Thread-2895 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:43:31,088 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=-1] 2024-11-07T12:43:31,088 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:43:31,089 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5] 2024-11-07T12:43:31,089 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47962, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:43:31,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5] 2024-11-07T12:43:31,089 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:43:31,090 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=-1] 2024-11-07T12:43:31,090 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:43:31,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-07T12:43:31,091 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5] 2024-11-07T12:43:31,091 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47978, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:43:31,091 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5] 2024-11-07T12:43:31,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-07T12:43:31,092 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5] 2024-11-07T12:43:31,094 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:43:31,095 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:43:31,096 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:43:31,101 DEBUG [Thread-2884 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:43:31,102 DEBUG [Thread-2884 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=-1] 2024-11-07T12:43:31,102 DEBUG [Thread-2884 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:43:31,103 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47984, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:43:31,105 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5] 2024-11-07T12:43:31,117 DEBUG [Thread-2878 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:43:31,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=-1] 2024-11-07T12:43:31,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:43:31,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:31,118 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 37eda6b0dad56caa0fa6c1ac8208d5b2 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-07T12:43:31,118 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47994, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:43:31,121 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5] 2024-11-07T12:43:31,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=A 2024-11-07T12:43:31,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:31,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=B 2024-11-07T12:43:31,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:31,122 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=C 2024-11-07T12:43:31,122 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:31,130 DEBUG [Thread-2887 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:43:31,131 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=-1] 2024-11-07T12:43:31,131 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:43:31,132 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48000, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:43:31,134 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5] 2024-11-07T12:43:31,136 DEBUG [Thread-2893 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:43:31,137 DEBUG [Thread-2893 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=-1] 2024-11-07T12:43:31,137 DEBUG [Thread-2893 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:43:31,139 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48002, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:43:31,142 DEBUG [Thread-2893 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5] 2024-11-07T12:43:31,162 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107434f71175c7540fa99959fe426b9066b_37eda6b0dad56caa0fa6c1ac8208d5b2 is 50, key is test_row_0/A:col10/1730983411111/Put/seqid=0 2024-11-07T12:43:31,175 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1730983411150, max timestamp=1730983411168Num uniques 859; 2024-11-07T12:43:31,175 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-07T12:43:31,177 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1730983411150, max timestamp=1730983411165Num uniques 859; 2024-11-07T12:43:31,177 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-07T12:43:31,178 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1730983411150, max timestamp=1730983411168Num uniques 859; 2024-11-07T12:43:31,178 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-07T12:43:31,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742482_1658 (size=12154) 2024-11-07T12:43:31,189 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:31,194 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107434f71175c7540fa99959fe426b9066b_37eda6b0dad56caa0fa6c1ac8208d5b2 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107434f71175c7540fa99959fe426b9066b_37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:31,194 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/c7dee467bb744760bc636cc8a9f6ffb4, store: [table=TestAcidGuarantees family=A region=37eda6b0dad56caa0fa6c1ac8208d5b2] 2024-11-07T12:43:31,195 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/c7dee467bb744760bc636cc8a9f6ffb4 is 175, key is test_row_0/A:col10/1730983411111/Put/seqid=0 2024-11-07T12:43:31,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742483_1659 (size=30955) 2024-11-07T12:43:31,198 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/c7dee467bb744760bc636cc8a9f6ffb4 2024-11-07T12:43:31,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-07T12:43:31,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:31,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47962 deadline: 1730983471205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:31,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:31,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1730983471207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:31,213 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:31,213 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:31,213 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:31,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:31,213 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:31,213 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:31,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47946 deadline: 1730983471208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:31,213 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:31,214 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:31,214 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:31,214 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:31,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:31,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47984 deadline: 1730983471211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:31,215 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:31,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:31,215 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:31,215 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:31,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47926 deadline: 1730983471213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:31,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:31,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:31,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:31,229 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/2360ab686d1a4ceba735eb4b44232082 is 50, key is test_row_0/B:col10/1730983411111/Put/seqid=0 2024-11-07T12:43:31,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742484_1660 (size=12001) 2024-11-07T12:43:31,243 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/2360ab686d1a4ceba735eb4b44232082 2024-11-07T12:43:31,264 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-07T12:43:31,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:31,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. as already flushing 2024-11-07T12:43:31,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:31,267 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:31,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:31,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:31,271 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/4b7f7a338fd14bb3abe360f9b651c790 is 50, key is test_row_0/C:col10/1730983411111/Put/seqid=0 2024-11-07T12:43:31,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742485_1661 (size=12001) 2024-11-07T12:43:31,324 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:31,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47962 deadline: 1730983471321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:31,324 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:31,324 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:31,324 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:31,324 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:31,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47946 deadline: 1730983471321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:31,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:31,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:31,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:31,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:31,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47984 deadline: 1730983471322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:31,326 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:31,326 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:31,326 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:31,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:31,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1730983471322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:31,326 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:31,326 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:31,326 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:31,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:31,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47926 deadline: 1730983471322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:31,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:31,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:31,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:31,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-07T12:43:31,421 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-07T12:43:31,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:31,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. as already flushing 2024-11-07T12:43:31,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:31,422 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:31,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:31,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:31,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:31,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47962 deadline: 1730983471532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:31,535 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:31,535 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:31,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:31,535 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:31,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47946 deadline: 1730983471532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:31,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:31,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1730983471532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:31,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:31,535 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:31,535 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:31,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47984 deadline: 1730983471532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:31,536 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:31,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:31,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47926 deadline: 1730983471532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:31,536 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:31,536 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:31,536 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:31,536 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:31,536 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:31,536 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:31,536 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:31,536 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:31,536 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:31,576 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-07T12:43:31,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:31,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. as already flushing 2024-11-07T12:43:31,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:31,576 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:31,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:31,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:31,708 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/4b7f7a338fd14bb3abe360f9b651c790 2024-11-07T12:43:31,715 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/c7dee467bb744760bc636cc8a9f6ffb4 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/c7dee467bb744760bc636cc8a9f6ffb4 2024-11-07T12:43:31,722 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/c7dee467bb744760bc636cc8a9f6ffb4, entries=150, sequenceid=15, filesize=30.2 K 2024-11-07T12:43:31,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-07T12:43:31,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/2360ab686d1a4ceba735eb4b44232082 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/2360ab686d1a4ceba735eb4b44232082 2024-11-07T12:43:31,727 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/2360ab686d1a4ceba735eb4b44232082, entries=150, sequenceid=15, filesize=11.7 K 2024-11-07T12:43:31,729 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/4b7f7a338fd14bb3abe360f9b651c790 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/4b7f7a338fd14bb3abe360f9b651c790 2024-11-07T12:43:31,733 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/4b7f7a338fd14bb3abe360f9b651c790, entries=150, sequenceid=15, filesize=11.7 K 2024-11-07T12:43:31,734 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 37eda6b0dad56caa0fa6c1ac8208d5b2 in 616ms, sequenceid=15, compaction requested=false 2024-11-07T12:43:31,734 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-07T12:43:31,735 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 37eda6b0dad56caa0fa6c1ac8208d5b2: 2024-11-07T12:43:31,735 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-07T12:43:31,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:31,735 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing 37eda6b0dad56caa0fa6c1ac8208d5b2 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-07T12:43:31,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=A 2024-11-07T12:43:31,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:43:31,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=B 2024-11-07T12:43:31,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:43:31,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=C 2024-11-07T12:43:31,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:43:31,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107db2ec0652cf746c5b3d1cf4311f3bc3e_37eda6b0dad56caa0fa6c1ac8208d5b2 is 50, key is test_row_0/A:col10/1730983411206/Put/seqid=0 2024-11-07T12:43:31,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742486_1662 (size=12154) 2024-11-07T12:43:31,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:31,755 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107db2ec0652cf746c5b3d1cf4311f3bc3e_37eda6b0dad56caa0fa6c1ac8208d5b2 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107db2ec0652cf746c5b3d1cf4311f3bc3e_37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:31,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/09780355c6714c6e9a59d31ee86f729c, store: [table=TestAcidGuarantees family=A region=37eda6b0dad56caa0fa6c1ac8208d5b2] 2024-11-07T12:43:31,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/09780355c6714c6e9a59d31ee86f729c is 175, key is test_row_0/A:col10/1730983411206/Put/seqid=0 2024-11-07T12:43:31,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742487_1663 (size=30955) 2024-11-07T12:43:31,782 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/09780355c6714c6e9a59d31ee86f729c 2024-11-07T12:43:31,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/b56fa0771a9a4a1d8591902acdf31104 is 50, key is test_row_0/B:col10/1730983411206/Put/seqid=0 2024-11-07T12:43:31,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742488_1664 (size=12001) 2024-11-07T12:43:31,807 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/b56fa0771a9a4a1d8591902acdf31104 2024-11-07T12:43:31,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/5377496dd313408cbba0cee4cd77166f is 50, key is test_row_0/C:col10/1730983411206/Put/seqid=0 2024-11-07T12:43:31,845 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. as already flushing 2024-11-07T12:43:31,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:31,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:31,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1730983471853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:31,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:31,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47962 deadline: 1730983471854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:31,860 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:31,860 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:31,860 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:31,860 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:31,860 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:31,860 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:31,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:31,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47926 deadline: 1730983471859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:31,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:31,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47946 deadline: 1730983471860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:31,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742489_1665 (size=12001) 2024-11-07T12:43:31,874 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:31,874 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:31,874 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:31,874 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:31,874 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:31,874 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:31,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:31,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47984 deadline: 1730983471860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:31,875 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/5377496dd313408cbba0cee4cd77166f 2024-11-07T12:43:31,876 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:31,876 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:31,876 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:31,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/09780355c6714c6e9a59d31ee86f729c as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/09780355c6714c6e9a59d31ee86f729c 2024-11-07T12:43:31,883 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/09780355c6714c6e9a59d31ee86f729c, entries=150, sequenceid=50, filesize=30.2 K 2024-11-07T12:43:31,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/b56fa0771a9a4a1d8591902acdf31104 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/b56fa0771a9a4a1d8591902acdf31104 2024-11-07T12:43:31,896 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/b56fa0771a9a4a1d8591902acdf31104, entries=150, sequenceid=50, filesize=11.7 K 2024-11-07T12:43:31,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/5377496dd313408cbba0cee4cd77166f as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/5377496dd313408cbba0cee4cd77166f 2024-11-07T12:43:31,902 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/5377496dd313408cbba0cee4cd77166f, entries=150, sequenceid=50, filesize=11.7 K 2024-11-07T12:43:31,912 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=67.09 KB/68700 for 37eda6b0dad56caa0fa6c1ac8208d5b2 in 177ms, sequenceid=50, compaction requested=false 2024-11-07T12:43:31,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 37eda6b0dad56caa0fa6c1ac8208d5b2: 2024-11-07T12:43:31,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:31,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-07T12:43:31,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-07T12:43:31,916 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-07T12:43:31,916 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 818 msec 2024-11-07T12:43:31,918 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 828 msec 2024-11-07T12:43:31,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:31,976 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 37eda6b0dad56caa0fa6c1ac8208d5b2 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-07T12:43:31,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=A 2024-11-07T12:43:31,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:31,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=B 2024-11-07T12:43:31,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:31,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=C 2024-11-07T12:43:31,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:31,998 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107f32715b3233e42598bd6632035723a3e_37eda6b0dad56caa0fa6c1ac8208d5b2 is 50, key is test_row_0/A:col10/1730983411976/Put/seqid=0 2024-11-07T12:43:32,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:32,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47962 deadline: 1730983472007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:32,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:32,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47946 deadline: 1730983472009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:32,022 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:32,022 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:32,022 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:32,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:32,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:32,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:32,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:32,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47984 deadline: 1730983472013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:32,024 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:32,024 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:32,025 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:32,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:32,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1730983472023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:32,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:32,032 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:32,032 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:32,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47926 deadline: 1730983472025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:32,032 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:32,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:32,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:32,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:32,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742490_1666 (size=14594) 2024-11-07T12:43:32,060 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:32,073 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107f32715b3233e42598bd6632035723a3e_37eda6b0dad56caa0fa6c1ac8208d5b2 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107f32715b3233e42598bd6632035723a3e_37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:32,075 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/8e40ed468304427499086b35a1d5d70d, store: [table=TestAcidGuarantees family=A region=37eda6b0dad56caa0fa6c1ac8208d5b2] 2024-11-07T12:43:32,075 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/8e40ed468304427499086b35a1d5d70d is 175, key is test_row_0/A:col10/1730983411976/Put/seqid=0 2024-11-07T12:43:32,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742491_1667 (size=39549) 2024-11-07T12:43:32,102 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=65, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/8e40ed468304427499086b35a1d5d70d 2024-11-07T12:43:32,110 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/6c669c34daef441aae2ec433b04d2efd is 50, key is test_row_0/B:col10/1730983411976/Put/seqid=0 2024-11-07T12:43:32,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:32,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47962 deadline: 1730983472131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:32,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:32,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47946 deadline: 1730983472132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:32,136 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:32,136 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:32,136 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:32,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:32,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:32,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47984 deadline: 1730983472132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:32,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:32,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:32,136 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:32,136 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:32,136 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:32,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742492_1668 (size=12001) 2024-11-07T12:43:32,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:32,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1730983472141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:32,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:32,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47926 deadline: 1730983472141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:32,147 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:32,147 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:32,148 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:32,148 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:32,148 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:32,147 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:32,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-07T12:43:32,232 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:43:32,233 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:43:32,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-07T12:43:32,235 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:43:32,236 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:43:32,236 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:43:32,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-07T12:43:32,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-07T12:43:32,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:32,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47962 deadline: 1730983472341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:32,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:32,345 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:32,345 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:32,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47946 deadline: 1730983472342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:32,345 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:32,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:32,345 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:32,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47984 deadline: 1730983472342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:32,345 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:32,345 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:32,345 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:32,345 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:32,345 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:32,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:32,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47926 deadline: 1730983472352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:32,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:32,357 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:32,357 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:32,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1730983472352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:32,357 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:32,357 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:32,357 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:32,357 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:32,389 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-07T12:43:32,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:32,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. as already flushing 2024-11-07T12:43:32,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:32,389 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:32,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:32,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:32,543 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-07T12:43:32,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:32,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. as already flushing 2024-11-07T12:43:32,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:32,544 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:32,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:32,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:32,547 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/6c669c34daef441aae2ec433b04d2efd 2024-11-07T12:43:32,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-07T12:43:32,574 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/4a1497e1972d4c4e9ec13bb61613848f is 50, key is test_row_0/C:col10/1730983411976/Put/seqid=0 2024-11-07T12:43:32,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742493_1669 (size=12001) 2024-11-07T12:43:32,600 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/4a1497e1972d4c4e9ec13bb61613848f 2024-11-07T12:43:32,606 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/8e40ed468304427499086b35a1d5d70d as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/8e40ed468304427499086b35a1d5d70d 2024-11-07T12:43:32,614 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/8e40ed468304427499086b35a1d5d70d, entries=200, sequenceid=65, filesize=38.6 K 2024-11-07T12:43:32,614 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/6c669c34daef441aae2ec433b04d2efd as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/6c669c34daef441aae2ec433b04d2efd 2024-11-07T12:43:32,619 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/6c669c34daef441aae2ec433b04d2efd, entries=150, sequenceid=65, filesize=11.7 K 2024-11-07T12:43:32,619 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/4a1497e1972d4c4e9ec13bb61613848f as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/4a1497e1972d4c4e9ec13bb61613848f 2024-11-07T12:43:32,624 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/4a1497e1972d4c4e9ec13bb61613848f, entries=150, sequenceid=65, filesize=11.7 K 2024-11-07T12:43:32,626 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 37eda6b0dad56caa0fa6c1ac8208d5b2 in 650ms, sequenceid=65, compaction requested=true 2024-11-07T12:43:32,626 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 37eda6b0dad56caa0fa6c1ac8208d5b2: 2024-11-07T12:43:32,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 37eda6b0dad56caa0fa6c1ac8208d5b2:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:43:32,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:32,626 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:43:32,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 37eda6b0dad56caa0fa6c1ac8208d5b2:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:43:32,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:32,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 37eda6b0dad56caa0fa6c1ac8208d5b2:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:43:32,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:43:32,626 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:43:32,627 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:43:32,627 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:43:32,627 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 37eda6b0dad56caa0fa6c1ac8208d5b2/B is initiating minor compaction (all files) 2024-11-07T12:43:32,628 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): 37eda6b0dad56caa0fa6c1ac8208d5b2/A is initiating minor compaction (all files) 2024-11-07T12:43:32,628 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 37eda6b0dad56caa0fa6c1ac8208d5b2/B in TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:32,628 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 37eda6b0dad56caa0fa6c1ac8208d5b2/A in TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:32,628 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/2360ab686d1a4ceba735eb4b44232082, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/b56fa0771a9a4a1d8591902acdf31104, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/6c669c34daef441aae2ec433b04d2efd] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp, totalSize=35.2 K 2024-11-07T12:43:32,628 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/c7dee467bb744760bc636cc8a9f6ffb4, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/09780355c6714c6e9a59d31ee86f729c, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/8e40ed468304427499086b35a1d5d70d] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp, totalSize=99.1 K 2024-11-07T12:43:32,628 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:32,628 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. files: [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/c7dee467bb744760bc636cc8a9f6ffb4, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/09780355c6714c6e9a59d31ee86f729c, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/8e40ed468304427499086b35a1d5d70d] 2024-11-07T12:43:32,628 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 2360ab686d1a4ceba735eb4b44232082, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1730983411093 2024-11-07T12:43:32,628 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting c7dee467bb744760bc636cc8a9f6ffb4, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1730983411093 2024-11-07T12:43:32,632 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09780355c6714c6e9a59d31ee86f729c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1730983411188 2024-11-07T12:43:32,632 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting b56fa0771a9a4a1d8591902acdf31104, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1730983411188 2024-11-07T12:43:32,632 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c669c34daef441aae2ec433b04d2efd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1730983411858 2024-11-07T12:43:32,632 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e40ed468304427499086b35a1d5d70d, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1730983411846 2024-11-07T12:43:32,648 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=37eda6b0dad56caa0fa6c1ac8208d5b2] 2024-11-07T12:43:32,655 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 37eda6b0dad56caa0fa6c1ac8208d5b2#B#compaction#575 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:43:32,656 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/022d206a82f54dd9ae5480872bf6da44 is 50, key is test_row_0/B:col10/1730983411976/Put/seqid=0 2024-11-07T12:43:32,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:32,658 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 37eda6b0dad56caa0fa6c1ac8208d5b2 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-07T12:43:32,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=A 2024-11-07T12:43:32,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:32,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=B 2024-11-07T12:43:32,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:32,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=C 2024-11-07T12:43:32,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:32,661 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241107f59f1fb4e8b047f9a135c4790cbea051_37eda6b0dad56caa0fa6c1ac8208d5b2 store=[table=TestAcidGuarantees family=A region=37eda6b0dad56caa0fa6c1ac8208d5b2] 2024-11-07T12:43:32,663 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241107f59f1fb4e8b047f9a135c4790cbea051_37eda6b0dad56caa0fa6c1ac8208d5b2, store=[table=TestAcidGuarantees family=A region=37eda6b0dad56caa0fa6c1ac8208d5b2] 2024-11-07T12:43:32,663 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107f59f1fb4e8b047f9a135c4790cbea051_37eda6b0dad56caa0fa6c1ac8208d5b2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=37eda6b0dad56caa0fa6c1ac8208d5b2] 2024-11-07T12:43:32,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:32,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47926 deadline: 1730983472671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:32,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:32,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:32,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:32,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:32,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47962 deadline: 1730983472674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:32,680 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:32,680 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:32,680 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:32,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:32,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47984 deadline: 1730983472678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:32,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:32,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:32,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:32,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:32,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47946 deadline: 1730983472679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:32,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:32,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1730983472680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:32,688 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:32,688 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:32,688 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:32,689 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:32,689 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:32,689 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:32,699 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-07T12:43:32,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:32,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. as already flushing 2024-11-07T12:43:32,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:32,700 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:32,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:32,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:32,701 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107b81f059cab444f298f11c51c5726594c_37eda6b0dad56caa0fa6c1ac8208d5b2 is 50, key is test_row_0/A:col10/1730983412007/Put/seqid=0 2024-11-07T12:43:32,726 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T12:43:32,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742495_1671 (size=4469) 2024-11-07T12:43:32,751 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 37eda6b0dad56caa0fa6c1ac8208d5b2#A#compaction#574 average throughput is 0.24 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:43:32,752 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/c8022b066d3e49fcada23459efbcf950 is 175, key is test_row_0/A:col10/1730983411976/Put/seqid=0 2024-11-07T12:43:32,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742494_1670 (size=12104) 2024-11-07T12:43:32,766 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/022d206a82f54dd9ae5480872bf6da44 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/022d206a82f54dd9ae5480872bf6da44 2024-11-07T12:43:32,771 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 37eda6b0dad56caa0fa6c1ac8208d5b2/B of 37eda6b0dad56caa0fa6c1ac8208d5b2 into 022d206a82f54dd9ae5480872bf6da44(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:43:32,771 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 37eda6b0dad56caa0fa6c1ac8208d5b2: 2024-11-07T12:43:32,771 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., storeName=37eda6b0dad56caa0fa6c1ac8208d5b2/B, priority=13, startTime=1730983412626; duration=0sec 2024-11-07T12:43:32,772 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:43:32,772 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 37eda6b0dad56caa0fa6c1ac8208d5b2:B 2024-11-07T12:43:32,772 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:43:32,772 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:43:32,772 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 37eda6b0dad56caa0fa6c1ac8208d5b2/C is initiating minor compaction (all files) 2024-11-07T12:43:32,773 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 37eda6b0dad56caa0fa6c1ac8208d5b2/C in TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:32,773 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/4b7f7a338fd14bb3abe360f9b651c790, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/5377496dd313408cbba0cee4cd77166f, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/4a1497e1972d4c4e9ec13bb61613848f] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp, totalSize=35.2 K 2024-11-07T12:43:32,773 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b7f7a338fd14bb3abe360f9b651c790, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1730983411093 2024-11-07T12:43:32,773 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 5377496dd313408cbba0cee4cd77166f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1730983411188 2024-11-07T12:43:32,774 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a1497e1972d4c4e9ec13bb61613848f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1730983411858 2024-11-07T12:43:32,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742496_1672 (size=14594) 2024-11-07T12:43:32,788 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:32,794 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107b81f059cab444f298f11c51c5726594c_37eda6b0dad56caa0fa6c1ac8208d5b2 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107b81f059cab444f298f11c51c5726594c_37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:32,795 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/308142f3672b41cab2eeac0a6eecef38, store: [table=TestAcidGuarantees family=A region=37eda6b0dad56caa0fa6c1ac8208d5b2] 2024-11-07T12:43:32,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:32,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47926 deadline: 1730983472785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:32,796 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/308142f3672b41cab2eeac0a6eecef38 is 175, key is test_row_0/A:col10/1730983412007/Put/seqid=0 2024-11-07T12:43:32,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:32,798 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:32,798 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:32,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:32,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47984 deadline: 1730983472792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:32,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:32,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47946 deadline: 1730983472792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:32,803 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:32,804 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:32,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:32,804 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:32,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:32,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47962 deadline: 1730983472793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:32,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1730983472793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:32,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:32,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:32,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:32,804 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:32,804 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:32,804 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:32,804 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:32,804 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:32,804 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:32,807 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 37eda6b0dad56caa0fa6c1ac8208d5b2#C#compaction#577 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:43:32,808 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/85f12dac2a8b486989909b9dc8ed1cf1 is 50, key is test_row_0/C:col10/1730983411976/Put/seqid=0 2024-11-07T12:43:32,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742497_1673 (size=31058) 2024-11-07T12:43:32,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742498_1674 (size=39549) 2024-11-07T12:43:32,849 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=85, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/308142f3672b41cab2eeac0a6eecef38 2024-11-07T12:43:32,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-07T12:43:32,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:32,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. as already flushing 2024-11-07T12:43:32,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:32,854 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:32,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:32,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:32,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-07T12:43:32,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742499_1675 (size=12104) 2024-11-07T12:43:32,891 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/3873223dbe5d4c49a920eab6632c61af is 50, key is test_row_0/B:col10/1730983412007/Put/seqid=0 2024-11-07T12:43:32,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742500_1676 (size=12001) 2024-11-07T12:43:32,897 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/3873223dbe5d4c49a920eab6632c61af 2024-11-07T12:43:32,906 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/7b971499c22e4654be525f40df839869 is 50, key is test_row_0/C:col10/1730983412007/Put/seqid=0 2024-11-07T12:43:32,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742501_1677 (size=12001) 2024-11-07T12:43:32,922 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/7b971499c22e4654be525f40df839869 2024-11-07T12:43:32,935 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/308142f3672b41cab2eeac0a6eecef38 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/308142f3672b41cab2eeac0a6eecef38 2024-11-07T12:43:32,939 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/308142f3672b41cab2eeac0a6eecef38, entries=200, sequenceid=85, filesize=38.6 K 2024-11-07T12:43:32,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/3873223dbe5d4c49a920eab6632c61af as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/3873223dbe5d4c49a920eab6632c61af 2024-11-07T12:43:32,947 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/3873223dbe5d4c49a920eab6632c61af, entries=150, sequenceid=85, filesize=11.7 K 2024-11-07T12:43:32,948 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/7b971499c22e4654be525f40df839869 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/7b971499c22e4654be525f40df839869 2024-11-07T12:43:32,959 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/7b971499c22e4654be525f40df839869, entries=150, sequenceid=85, filesize=11.7 K 2024-11-07T12:43:32,960 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 37eda6b0dad56caa0fa6c1ac8208d5b2 in 302ms, sequenceid=85, compaction requested=false 2024-11-07T12:43:32,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 37eda6b0dad56caa0fa6c1ac8208d5b2: 2024-11-07T12:43:33,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:33,007 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-07T12:43:33,007 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 37eda6b0dad56caa0fa6c1ac8208d5b2 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-07T12:43:33,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:33,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. as already flushing 2024-11-07T12:43:33,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:33,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=A 2024-11-07T12:43:33,007 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:33,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:33,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=B 2024-11-07T12:43:33,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:33,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:33,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=C 2024-11-07T12:43:33,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:33,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:33,025 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107f902386e4a3c4e07bc76f2fad14b19f8_37eda6b0dad56caa0fa6c1ac8208d5b2 is 50, key is test_row_0/A:col10/1730983412671/Put/seqid=0 2024-11-07T12:43:33,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742502_1678 (size=14594) 2024-11-07T12:43:33,035 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:33,042 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107f902386e4a3c4e07bc76f2fad14b19f8_37eda6b0dad56caa0fa6c1ac8208d5b2 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107f902386e4a3c4e07bc76f2fad14b19f8_37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:33,047 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/ac18115502bd40fab50c1f0a689a6759, store: [table=TestAcidGuarantees family=A region=37eda6b0dad56caa0fa6c1ac8208d5b2] 2024-11-07T12:43:33,048 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/ac18115502bd40fab50c1f0a689a6759 is 175, key is test_row_0/A:col10/1730983412671/Put/seqid=0 2024-11-07T12:43:33,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:33,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:33,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47984 deadline: 1730983473048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:33,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47926 deadline: 1730983473045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:33,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:33,058 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:33,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:33,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:33,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47946 deadline: 1730983473052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:33,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:33,058 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:33,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1730983473049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:33,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47962 deadline: 1730983473053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:33,059 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:33,059 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:33,059 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:33,059 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:33,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:33,059 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:33,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:33,059 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:33,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:33,059 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:33,059 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:33,059 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:33,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742503_1679 (size=39549) 2024-11-07T12:43:33,064 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=100, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/ac18115502bd40fab50c1f0a689a6759 2024-11-07T12:43:33,094 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/f0437c51a8ab4711b1b975bacd5a9414 is 50, key is test_row_0/B:col10/1730983412671/Put/seqid=0 2024-11-07T12:43:33,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742504_1680 (size=12001) 2024-11-07T12:43:33,167 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-07T12:43:33,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:33,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. as already flushing 2024-11-07T12:43:33,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:33,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:33,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47926 deadline: 1730983473162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:33,167 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:33,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:33,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:33,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:33,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1730983473162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:33,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:33,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:33,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:33,168 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:33,168 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:33,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47984 deadline: 1730983473162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:33,168 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:33,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:33,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47946 deadline: 1730983473162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:33,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:33,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:33,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47962 deadline: 1730983473163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:33,169 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:33,169 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:33,169 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:33,169 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:33,169 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:33,169 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:33,170 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:33,170 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:33,170 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:33,243 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/c8022b066d3e49fcada23459efbcf950 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/c8022b066d3e49fcada23459efbcf950 2024-11-07T12:43:33,247 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 37eda6b0dad56caa0fa6c1ac8208d5b2/A of 37eda6b0dad56caa0fa6c1ac8208d5b2 into c8022b066d3e49fcada23459efbcf950(size=30.3 K), total size for store is 69.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:43:33,247 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 37eda6b0dad56caa0fa6c1ac8208d5b2: 2024-11-07T12:43:33,247 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., storeName=37eda6b0dad56caa0fa6c1ac8208d5b2/A, priority=13, startTime=1730983412626; duration=0sec 2024-11-07T12:43:33,247 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:33,247 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 37eda6b0dad56caa0fa6c1ac8208d5b2:A 2024-11-07T12:43:33,280 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/85f12dac2a8b486989909b9dc8ed1cf1 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/85f12dac2a8b486989909b9dc8ed1cf1 2024-11-07T12:43:33,285 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 37eda6b0dad56caa0fa6c1ac8208d5b2/C of 37eda6b0dad56caa0fa6c1ac8208d5b2 into 85f12dac2a8b486989909b9dc8ed1cf1(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:43:33,285 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 37eda6b0dad56caa0fa6c1ac8208d5b2: 2024-11-07T12:43:33,285 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., storeName=37eda6b0dad56caa0fa6c1ac8208d5b2/C, priority=13, startTime=1730983412626; duration=0sec 2024-11-07T12:43:33,285 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:33,285 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 37eda6b0dad56caa0fa6c1ac8208d5b2:C 2024-11-07T12:43:33,321 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-07T12:43:33,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:33,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. as already flushing 2024-11-07T12:43:33,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:33,321 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:33,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:33,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:33,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-07T12:43:33,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:33,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1730983473372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:33,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:33,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:33,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47984 deadline: 1730983473372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:33,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47946 deadline: 1730983473372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:33,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:33,377 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:33,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47926 deadline: 1730983473372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:33,378 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:33,378 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:33,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:33,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:33,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:33,383 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:33,383 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:33,383 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:33,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:33,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:33,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:33,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:33,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47962 deadline: 1730983473382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:33,384 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:33,384 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:33,384 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:33,477 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-07T12:43:33,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:33,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. as already flushing 2024-11-07T12:43:33,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:33,477 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:33,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:33,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:33,527 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/f0437c51a8ab4711b1b975bacd5a9414 2024-11-07T12:43:33,546 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/e4d489060d7e41c8953a042e6f024561 is 50, key is test_row_0/C:col10/1730983412671/Put/seqid=0 2024-11-07T12:43:33,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742505_1681 (size=12001) 2024-11-07T12:43:33,558 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/e4d489060d7e41c8953a042e6f024561 2024-11-07T12:43:33,564 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/ac18115502bd40fab50c1f0a689a6759 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/ac18115502bd40fab50c1f0a689a6759 2024-11-07T12:43:33,568 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/ac18115502bd40fab50c1f0a689a6759, entries=200, sequenceid=100, filesize=38.6 K 2024-11-07T12:43:33,570 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/f0437c51a8ab4711b1b975bacd5a9414 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/f0437c51a8ab4711b1b975bacd5a9414 2024-11-07T12:43:33,575 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/f0437c51a8ab4711b1b975bacd5a9414, entries=150, sequenceid=100, filesize=11.7 K 2024-11-07T12:43:33,577 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/e4d489060d7e41c8953a042e6f024561 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/e4d489060d7e41c8953a042e6f024561 2024-11-07T12:43:33,582 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/e4d489060d7e41c8953a042e6f024561, entries=150, sequenceid=100, filesize=11.7 K 2024-11-07T12:43:33,583 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 37eda6b0dad56caa0fa6c1ac8208d5b2 in 576ms, sequenceid=100, compaction requested=true 2024-11-07T12:43:33,583 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 37eda6b0dad56caa0fa6c1ac8208d5b2: 2024-11-07T12:43:33,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 37eda6b0dad56caa0fa6c1ac8208d5b2:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:43:33,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:33,583 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:43:33,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 37eda6b0dad56caa0fa6c1ac8208d5b2:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:43:33,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:33,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 37eda6b0dad56caa0fa6c1ac8208d5b2:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:43:33,583 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:43:33,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:43:33,584 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110156 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:43:33,584 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): 37eda6b0dad56caa0fa6c1ac8208d5b2/A is initiating minor compaction (all files) 2024-11-07T12:43:33,584 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 37eda6b0dad56caa0fa6c1ac8208d5b2/A in TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:33,584 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/c8022b066d3e49fcada23459efbcf950, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/308142f3672b41cab2eeac0a6eecef38, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/ac18115502bd40fab50c1f0a689a6759] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp, totalSize=107.6 K 2024-11-07T12:43:33,584 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:33,585 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. files: [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/c8022b066d3e49fcada23459efbcf950, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/308142f3672b41cab2eeac0a6eecef38, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/ac18115502bd40fab50c1f0a689a6759] 2024-11-07T12:43:33,585 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting c8022b066d3e49fcada23459efbcf950, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1730983411858 2024-11-07T12:43:33,585 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 308142f3672b41cab2eeac0a6eecef38, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1730983412004 2024-11-07T12:43:33,586 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac18115502bd40fab50c1f0a689a6759, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1730983412671 2024-11-07T12:43:33,587 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:43:33,587 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 37eda6b0dad56caa0fa6c1ac8208d5b2/B is initiating minor compaction (all files) 2024-11-07T12:43:33,587 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 37eda6b0dad56caa0fa6c1ac8208d5b2/B in TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:33,587 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/022d206a82f54dd9ae5480872bf6da44, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/3873223dbe5d4c49a920eab6632c61af, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/f0437c51a8ab4711b1b975bacd5a9414] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp, totalSize=35.3 K 2024-11-07T12:43:33,587 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 022d206a82f54dd9ae5480872bf6da44, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1730983411858 2024-11-07T12:43:33,589 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 3873223dbe5d4c49a920eab6632c61af, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1730983412004 2024-11-07T12:43:33,589 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting f0437c51a8ab4711b1b975bacd5a9414, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1730983412671 2024-11-07T12:43:33,594 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=37eda6b0dad56caa0fa6c1ac8208d5b2] 2024-11-07T12:43:33,605 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411075c5c2bb2326b4a2bbdb2166b456e66dd_37eda6b0dad56caa0fa6c1ac8208d5b2 store=[table=TestAcidGuarantees family=A region=37eda6b0dad56caa0fa6c1ac8208d5b2] 2024-11-07T12:43:33,607 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411075c5c2bb2326b4a2bbdb2166b456e66dd_37eda6b0dad56caa0fa6c1ac8208d5b2, store=[table=TestAcidGuarantees family=A region=37eda6b0dad56caa0fa6c1ac8208d5b2] 2024-11-07T12:43:33,607 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411075c5c2bb2326b4a2bbdb2166b456e66dd_37eda6b0dad56caa0fa6c1ac8208d5b2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=37eda6b0dad56caa0fa6c1ac8208d5b2] 2024-11-07T12:43:33,623 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 37eda6b0dad56caa0fa6c1ac8208d5b2#B#compaction#584 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:43:33,623 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/1f827ba940614c04b450a74cdef0b626 is 50, key is test_row_0/B:col10/1730983412671/Put/seqid=0 2024-11-07T12:43:33,630 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-07T12:43:33,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:33,631 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing 37eda6b0dad56caa0fa6c1ac8208d5b2 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-07T12:43:33,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=A 2024-11-07T12:43:33,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:33,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=B 2024-11-07T12:43:33,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:33,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=C 2024-11-07T12:43:33,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:33,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742506_1682 (size=4469) 2024-11-07T12:43:33,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742507_1683 (size=12207) 2024-11-07T12:43:33,661 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 37eda6b0dad56caa0fa6c1ac8208d5b2#A#compaction#583 average throughput is 0.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:43:33,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411074d48a49edfd74746bc1497fc056bff2d_37eda6b0dad56caa0fa6c1ac8208d5b2 is 50, key is test_row_0/A:col10/1730983413047/Put/seqid=0 2024-11-07T12:43:33,664 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/43d6ff7d297e4a8fa7ef199c86972bf2 is 175, key is test_row_0/A:col10/1730983412671/Put/seqid=0 2024-11-07T12:43:33,673 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/1f827ba940614c04b450a74cdef0b626 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/1f827ba940614c04b450a74cdef0b626 2024-11-07T12:43:33,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742508_1684 (size=31161) 2024-11-07T12:43:33,680 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 37eda6b0dad56caa0fa6c1ac8208d5b2/B of 37eda6b0dad56caa0fa6c1ac8208d5b2 into 1f827ba940614c04b450a74cdef0b626(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:43:33,680 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 37eda6b0dad56caa0fa6c1ac8208d5b2: 2024-11-07T12:43:33,680 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., storeName=37eda6b0dad56caa0fa6c1ac8208d5b2/B, priority=13, startTime=1730983413583; duration=0sec 2024-11-07T12:43:33,680 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:43:33,680 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 37eda6b0dad56caa0fa6c1ac8208d5b2:B 2024-11-07T12:43:33,681 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:43:33,681 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:43:33,681 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 37eda6b0dad56caa0fa6c1ac8208d5b2/C is initiating minor compaction (all files) 2024-11-07T12:43:33,682 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 37eda6b0dad56caa0fa6c1ac8208d5b2/C in TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:33,682 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/85f12dac2a8b486989909b9dc8ed1cf1, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/7b971499c22e4654be525f40df839869, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/e4d489060d7e41c8953a042e6f024561] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp, totalSize=35.3 K 2024-11-07T12:43:33,682 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 85f12dac2a8b486989909b9dc8ed1cf1, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1730983411858 2024-11-07T12:43:33,683 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b971499c22e4654be525f40df839869, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1730983412004 2024-11-07T12:43:33,683 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting e4d489060d7e41c8953a042e6f024561, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1730983412671 2024-11-07T12:43:33,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:33,685 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. as already flushing 2024-11-07T12:43:33,695 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 37eda6b0dad56caa0fa6c1ac8208d5b2#C#compaction#586 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:43:33,696 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/6b089f6269ed47d69fc023d6f626cd3e is 50, key is test_row_0/C:col10/1730983412671/Put/seqid=0 2024-11-07T12:43:33,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742509_1685 (size=12154) 2024-11-07T12:43:33,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:33,702 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/43d6ff7d297e4a8fa7ef199c86972bf2 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/43d6ff7d297e4a8fa7ef199c86972bf2 2024-11-07T12:43:33,704 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411074d48a49edfd74746bc1497fc056bff2d_37eda6b0dad56caa0fa6c1ac8208d5b2 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411074d48a49edfd74746bc1497fc056bff2d_37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:33,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/ff11c6e977764ce585ca73a34b014061, store: [table=TestAcidGuarantees family=A region=37eda6b0dad56caa0fa6c1ac8208d5b2] 2024-11-07T12:43:33,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/ff11c6e977764ce585ca73a34b014061 is 175, key is test_row_0/A:col10/1730983413047/Put/seqid=0 2024-11-07T12:43:33,707 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 37eda6b0dad56caa0fa6c1ac8208d5b2/A of 37eda6b0dad56caa0fa6c1ac8208d5b2 into 43d6ff7d297e4a8fa7ef199c86972bf2(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:43:33,707 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 37eda6b0dad56caa0fa6c1ac8208d5b2: 2024-11-07T12:43:33,707 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., storeName=37eda6b0dad56caa0fa6c1ac8208d5b2/A, priority=13, startTime=1730983413583; duration=0sec 2024-11-07T12:43:33,708 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:33,708 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 37eda6b0dad56caa0fa6c1ac8208d5b2:A 2024-11-07T12:43:33,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742510_1686 (size=12207) 2024-11-07T12:43:33,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742511_1687 (size=30955) 2024-11-07T12:43:33,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:33,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47984 deadline: 1730983473709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:33,716 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=120, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/ff11c6e977764ce585ca73a34b014061 2024-11-07T12:43:33,716 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:33,716 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:33,716 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:33,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:33,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47946 deadline: 1730983473712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:33,720 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:33,720 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:33,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:33,720 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:33,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47926 deadline: 1730983473714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:33,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:33,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:33,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:33,721 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:33,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47962 deadline: 1730983473714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:33,721 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:33,721 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:33,721 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:33,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1730983473716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:33,722 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:33,722 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/6b089f6269ed47d69fc023d6f626cd3e as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/6b089f6269ed47d69fc023d6f626cd3e 2024-11-07T12:43:33,723 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:33,723 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:33,723 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:33,729 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 37eda6b0dad56caa0fa6c1ac8208d5b2/C of 37eda6b0dad56caa0fa6c1ac8208d5b2 into 6b089f6269ed47d69fc023d6f626cd3e(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:43:33,729 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 37eda6b0dad56caa0fa6c1ac8208d5b2: 2024-11-07T12:43:33,729 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., storeName=37eda6b0dad56caa0fa6c1ac8208d5b2/C, priority=13, startTime=1730983413583; duration=0sec 2024-11-07T12:43:33,729 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:33,729 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 37eda6b0dad56caa0fa6c1ac8208d5b2:C 2024-11-07T12:43:33,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/7011b00d08b741509e02be8e21885370 is 50, key is test_row_0/B:col10/1730983413047/Put/seqid=0 2024-11-07T12:43:33,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742512_1688 (size=12001) 2024-11-07T12:43:33,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:33,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47984 deadline: 1730983473822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:33,825 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:33,825 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:33,825 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:33,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:33,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47926 deadline: 1730983473832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:33,834 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:33,834 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:33,834 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:33,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:33,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1730983473833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:33,837 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:33,837 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:33,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:33,837 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:33,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47946 deadline: 1730983473833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:33,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:33,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:33,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:33,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:33,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47962 deadline: 1730983473833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:33,838 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:33,838 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:33,838 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:34,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:34,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47984 deadline: 1730983474031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:34,035 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:34,035 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:34,035 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:34,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:34,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47926 deadline: 1730983474042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:34,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:34,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:34,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:34,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:34,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1730983474042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:34,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:34,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47946 deadline: 1730983474042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:34,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:34,046 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:34,046 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:34,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47962 deadline: 1730983474042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:34,046 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:34,046 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:34,046 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:34,046 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:34,046 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:34,046 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:34,046 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:34,153 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/7011b00d08b741509e02be8e21885370 2024-11-07T12:43:34,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/3fd6ce875c6a4a39b58e15d863c0f517 is 50, key is test_row_0/C:col10/1730983413047/Put/seqid=0 2024-11-07T12:43:34,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742513_1689 (size=12001) 2024-11-07T12:43:34,211 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/3fd6ce875c6a4a39b58e15d863c0f517 2024-11-07T12:43:34,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/ff11c6e977764ce585ca73a34b014061 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/ff11c6e977764ce585ca73a34b014061 2024-11-07T12:43:34,241 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/ff11c6e977764ce585ca73a34b014061, entries=150, sequenceid=120, filesize=30.2 K 2024-11-07T12:43:34,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/7011b00d08b741509e02be8e21885370 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/7011b00d08b741509e02be8e21885370 2024-11-07T12:43:34,251 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/7011b00d08b741509e02be8e21885370, entries=150, sequenceid=120, filesize=11.7 K 2024-11-07T12:43:34,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/3fd6ce875c6a4a39b58e15d863c0f517 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/3fd6ce875c6a4a39b58e15d863c0f517 2024-11-07T12:43:34,258 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/3fd6ce875c6a4a39b58e15d863c0f517, entries=150, sequenceid=120, filesize=11.7 K 2024-11-07T12:43:34,259 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 37eda6b0dad56caa0fa6c1ac8208d5b2 in 628ms, sequenceid=120, compaction requested=false 2024-11-07T12:43:34,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 37eda6b0dad56caa0fa6c1ac8208d5b2: 2024-11-07T12:43:34,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:34,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-07T12:43:34,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-07T12:43:34,263 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-07T12:43:34,263 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0250 sec 2024-11-07T12:43:34,266 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 2.0310 sec 2024-11-07T12:43:34,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:34,345 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 37eda6b0dad56caa0fa6c1ac8208d5b2 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-07T12:43:34,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=A 2024-11-07T12:43:34,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:34,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=B 2024-11-07T12:43:34,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:34,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=C 2024-11-07T12:43:34,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:34,357 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411074401f596178548a99548bc06acf057d3_37eda6b0dad56caa0fa6c1ac8208d5b2 is 50, key is test_row_0/A:col10/1730983413710/Put/seqid=0 2024-11-07T12:43:34,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742514_1690 (size=14744) 2024-11-07T12:43:34,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-07T12:43:34,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:34,385 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:34,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47946 deadline: 1730983474375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:34,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47984 deadline: 1730983474374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:34,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:34,385 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:34,385 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:34,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1730983474383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:34,385 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:34,385 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:34,387 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:34,387 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:34,387 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:34,387 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:34,387 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:34,389 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:43:34,392 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:43:34,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:34,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47926 deadline: 1730983474386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:34,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-07T12:43:34,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:34,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47962 deadline: 1730983474387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:34,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:34,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:34,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:34,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-07T12:43:34,395 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:43:34,396 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:43:34,396 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:43:34,396 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:34,399 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:34,399 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:34,399 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:34,406 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411074401f596178548a99548bc06acf057d3_37eda6b0dad56caa0fa6c1ac8208d5b2 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411074401f596178548a99548bc06acf057d3_37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:34,407 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/18951ee79df04451be85a8704467ede8, store: [table=TestAcidGuarantees family=A region=37eda6b0dad56caa0fa6c1ac8208d5b2] 2024-11-07T12:43:34,408 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/18951ee79df04451be85a8704467ede8 is 175, key is test_row_0/A:col10/1730983413710/Put/seqid=0 2024-11-07T12:43:34,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742515_1691 (size=39699) 2024-11-07T12:43:34,449 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=138, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/18951ee79df04451be85a8704467ede8 2024-11-07T12:43:34,464 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/c42b46cdd0654982a3e3ab38107e54da is 50, key is test_row_0/B:col10/1730983413710/Put/seqid=0 2024-11-07T12:43:34,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:34,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47946 deadline: 1730983474492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:34,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:34,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47984 deadline: 1730983474492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:34,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:34,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1730983474492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:34,495 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:34,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:34,496 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:34,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:34,496 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:34,496 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:34,496 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:34,496 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:34,496 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:34,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-07T12:43:34,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:34,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47926 deadline: 1730983474502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:34,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:34,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47962 deadline: 1730983474502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:34,508 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:34,508 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:34,508 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:34,511 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:34,511 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:34,511 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:34,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742516_1692 (size=12101) 2024-11-07T12:43:34,553 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-07T12:43:34,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:34,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. as already flushing 2024-11-07T12:43:34,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:34,553 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:34,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:34,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:34,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:34,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1730983474702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:34,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:34,705 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:34,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47946 deadline: 1730983474702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:34,705 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:34,705 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:34,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:34,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47984 deadline: 1730983474702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:34,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:34,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:34,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:34,706 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:34,706 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:34,706 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:34,708 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-07T12:43:34,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:34,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. as already flushing 2024-11-07T12:43:34,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:34,709 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:34,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:34,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:34,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-07T12:43:34,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:34,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47962 deadline: 1730983474712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:34,716 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:34,716 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:34,716 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:34,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:34,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47926 deadline: 1730983474722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:34,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:34,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:34,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:34,861 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-07T12:43:34,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:34,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. as already flushing 2024-11-07T12:43:34,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:34,862 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:34,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:34,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:34,919 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/c42b46cdd0654982a3e3ab38107e54da 2024-11-07T12:43:34,934 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/405c0221dfef42f9b42a6384fbb6acdc is 50, key is test_row_0/C:col10/1730983413710/Put/seqid=0 2024-11-07T12:43:34,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742517_1693 (size=12101) 2024-11-07T12:43:34,951 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/405c0221dfef42f9b42a6384fbb6acdc 2024-11-07T12:43:34,956 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/18951ee79df04451be85a8704467ede8 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/18951ee79df04451be85a8704467ede8 2024-11-07T12:43:34,969 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/18951ee79df04451be85a8704467ede8, entries=200, sequenceid=138, filesize=38.8 K 2024-11-07T12:43:34,970 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/c42b46cdd0654982a3e3ab38107e54da as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/c42b46cdd0654982a3e3ab38107e54da 2024-11-07T12:43:34,973 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/c42b46cdd0654982a3e3ab38107e54da, entries=150, sequenceid=138, filesize=11.8 K 2024-11-07T12:43:34,974 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/405c0221dfef42f9b42a6384fbb6acdc as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/405c0221dfef42f9b42a6384fbb6acdc 2024-11-07T12:43:34,977 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/405c0221dfef42f9b42a6384fbb6acdc, entries=150, sequenceid=138, filesize=11.8 K 2024-11-07T12:43:34,978 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 37eda6b0dad56caa0fa6c1ac8208d5b2 in 634ms, sequenceid=138, compaction requested=true 2024-11-07T12:43:34,978 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 37eda6b0dad56caa0fa6c1ac8208d5b2: 2024-11-07T12:43:34,979 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:43:34,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 37eda6b0dad56caa0fa6c1ac8208d5b2:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:43:34,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:34,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 37eda6b0dad56caa0fa6c1ac8208d5b2:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:43:34,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:34,979 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101815 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:43:34,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 37eda6b0dad56caa0fa6c1ac8208d5b2:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:43:34,979 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:43:34,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:43:34,979 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): 37eda6b0dad56caa0fa6c1ac8208d5b2/A is initiating minor compaction (all files) 2024-11-07T12:43:34,979 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 37eda6b0dad56caa0fa6c1ac8208d5b2/A in TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:34,980 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/43d6ff7d297e4a8fa7ef199c86972bf2, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/ff11c6e977764ce585ca73a34b014061, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/18951ee79df04451be85a8704467ede8] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp, totalSize=99.4 K 2024-11-07T12:43:34,980 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:34,980 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. files: [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/43d6ff7d297e4a8fa7ef199c86972bf2, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/ff11c6e977764ce585ca73a34b014061, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/18951ee79df04451be85a8704467ede8] 2024-11-07T12:43:34,980 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43d6ff7d297e4a8fa7ef199c86972bf2, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1730983412671 2024-11-07T12:43:34,980 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff11c6e977764ce585ca73a34b014061, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1730983413047 2024-11-07T12:43:34,980 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:43:34,980 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 37eda6b0dad56caa0fa6c1ac8208d5b2/B is initiating minor compaction (all files) 2024-11-07T12:43:34,980 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 37eda6b0dad56caa0fa6c1ac8208d5b2/B in TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:34,981 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/1f827ba940614c04b450a74cdef0b626, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/7011b00d08b741509e02be8e21885370, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/c42b46cdd0654982a3e3ab38107e54da] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp, totalSize=35.5 K 2024-11-07T12:43:34,981 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18951ee79df04451be85a8704467ede8, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1730983413696 2024-11-07T12:43:34,981 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f827ba940614c04b450a74cdef0b626, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1730983412671 2024-11-07T12:43:34,981 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 7011b00d08b741509e02be8e21885370, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1730983413047 2024-11-07T12:43:34,982 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting c42b46cdd0654982a3e3ab38107e54da, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1730983413696 2024-11-07T12:43:34,989 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=37eda6b0dad56caa0fa6c1ac8208d5b2] 2024-11-07T12:43:35,002 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 37eda6b0dad56caa0fa6c1ac8208d5b2#B#compaction#593 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:43:35,003 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/d08906e3ea444fe6a662fc09b0c48b13 is 50, key is test_row_0/B:col10/1730983413710/Put/seqid=0 2024-11-07T12:43:35,015 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-07T12:43:35,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:35,015 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing 37eda6b0dad56caa0fa6c1ac8208d5b2 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-07T12:43:35,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=A 2024-11-07T12:43:35,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:35,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=B 2024-11-07T12:43:35,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:35,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=C 2024-11-07T12:43:35,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:35,017 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. as already flushing 2024-11-07T12:43:35,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:35,021 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241107232466eb8cf34b818c8bc5008be9b541_37eda6b0dad56caa0fa6c1ac8208d5b2 store=[table=TestAcidGuarantees family=A region=37eda6b0dad56caa0fa6c1ac8208d5b2] 2024-11-07T12:43:35,023 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241107232466eb8cf34b818c8bc5008be9b541_37eda6b0dad56caa0fa6c1ac8208d5b2, store=[table=TestAcidGuarantees family=A region=37eda6b0dad56caa0fa6c1ac8208d5b2] 2024-11-07T12:43:35,023 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107232466eb8cf34b818c8bc5008be9b541_37eda6b0dad56caa0fa6c1ac8208d5b2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=37eda6b0dad56caa0fa6c1ac8208d5b2] 2024-11-07T12:43:35,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-07T12:43:35,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:35,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1730983475037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:35,045 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:35,045 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:35,045 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:35,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:35,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47946 deadline: 1730983475039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:35,046 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:35,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:35,047 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:35,047 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:35,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47926 deadline: 1730983475040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:35,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:35,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:35,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:35,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:35,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47962 deadline: 1730983475043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:35,052 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:35,052 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:35,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:35,052 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:35,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47984 deadline: 1730983475044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:35,053 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:35,053 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:35,053 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:35,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742518_1694 (size=12409) 2024-11-07T12:43:35,078 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/d08906e3ea444fe6a662fc09b0c48b13 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/d08906e3ea444fe6a662fc09b0c48b13 2024-11-07T12:43:35,082 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 37eda6b0dad56caa0fa6c1ac8208d5b2/B of 37eda6b0dad56caa0fa6c1ac8208d5b2 into d08906e3ea444fe6a662fc09b0c48b13(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:43:35,082 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 37eda6b0dad56caa0fa6c1ac8208d5b2: 2024-11-07T12:43:35,083 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., storeName=37eda6b0dad56caa0fa6c1ac8208d5b2/B, priority=13, startTime=1730983414979; duration=0sec 2024-11-07T12:43:35,083 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:43:35,083 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 37eda6b0dad56caa0fa6c1ac8208d5b2:B 2024-11-07T12:43:35,083 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:43:35,084 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:43:35,084 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 37eda6b0dad56caa0fa6c1ac8208d5b2/C is initiating minor compaction (all files) 2024-11-07T12:43:35,084 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 37eda6b0dad56caa0fa6c1ac8208d5b2/C in TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:35,084 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/6b089f6269ed47d69fc023d6f626cd3e, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/3fd6ce875c6a4a39b58e15d863c0f517, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/405c0221dfef42f9b42a6384fbb6acdc] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp, totalSize=35.5 K 2024-11-07T12:43:35,084 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b089f6269ed47d69fc023d6f626cd3e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1730983412671 2024-11-07T12:43:35,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742519_1695 (size=4469) 2024-11-07T12:43:35,085 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 3fd6ce875c6a4a39b58e15d863c0f517, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1730983413047 2024-11-07T12:43:35,085 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 405c0221dfef42f9b42a6384fbb6acdc, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1730983413696 2024-11-07T12:43:35,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411076d0ddb3c23ac4612bd4708a8c04a467e_37eda6b0dad56caa0fa6c1ac8208d5b2 is 50, key is test_row_0/A:col10/1730983414382/Put/seqid=0 2024-11-07T12:43:35,115 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 37eda6b0dad56caa0fa6c1ac8208d5b2#C#compaction#595 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:43:35,115 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/3a5151d403bd42c683df1863703c8832 is 50, key is test_row_0/C:col10/1730983413710/Put/seqid=0 2024-11-07T12:43:35,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742520_1696 (size=12304) 2024-11-07T12:43:35,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:35,152 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411076d0ddb3c23ac4612bd4708a8c04a467e_37eda6b0dad56caa0fa6c1ac8208d5b2 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411076d0ddb3c23ac4612bd4708a8c04a467e_37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:35,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/ad853594617743f9b3f8ccc4d4633d23, store: [table=TestAcidGuarantees family=A region=37eda6b0dad56caa0fa6c1ac8208d5b2] 2024-11-07T12:43:35,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/ad853594617743f9b3f8ccc4d4633d23 is 175, key is test_row_0/A:col10/1730983414382/Put/seqid=0 2024-11-07T12:43:35,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:35,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1730983475152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:35,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:35,157 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:35,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47926 deadline: 1730983475152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:35,157 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:35,157 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:35,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:35,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:35,158 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:35,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:35,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47946 deadline: 1730983475152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:35,158 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:35,158 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:35,158 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:35,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742521_1697 (size=12409) 2024-11-07T12:43:35,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:35,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47984 deadline: 1730983475169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:35,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742522_1698 (size=31105) 2024-11-07T12:43:35,174 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:35,174 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:35,174 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:35,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:35,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47962 deadline: 1730983475169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:35,175 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:35,175 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:35,175 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:35,182 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=155, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/ad853594617743f9b3f8ccc4d4633d23 2024-11-07T12:43:35,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/4cd28318f27c4b3893ca37805a6b557f is 50, key is test_row_0/B:col10/1730983414382/Put/seqid=0 2024-11-07T12:43:35,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742523_1699 (size=12151) 2024-11-07T12:43:35,225 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/4cd28318f27c4b3893ca37805a6b557f 2024-11-07T12:43:35,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/8a0a74a3b90544c99771294d7a658410 is 50, key is test_row_0/C:col10/1730983414382/Put/seqid=0 2024-11-07T12:43:35,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742524_1700 (size=12151) 2024-11-07T12:43:35,271 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/8a0a74a3b90544c99771294d7a658410 2024-11-07T12:43:35,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/ad853594617743f9b3f8ccc4d4633d23 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/ad853594617743f9b3f8ccc4d4633d23 2024-11-07T12:43:35,283 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/ad853594617743f9b3f8ccc4d4633d23, entries=150, sequenceid=155, filesize=30.4 K 2024-11-07T12:43:35,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/4cd28318f27c4b3893ca37805a6b557f as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/4cd28318f27c4b3893ca37805a6b557f 2024-11-07T12:43:35,292 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/4cd28318f27c4b3893ca37805a6b557f, entries=150, sequenceid=155, filesize=11.9 K 2024-11-07T12:43:35,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/8a0a74a3b90544c99771294d7a658410 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/8a0a74a3b90544c99771294d7a658410 2024-11-07T12:43:35,297 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/8a0a74a3b90544c99771294d7a658410, entries=150, sequenceid=155, filesize=11.9 K 2024-11-07T12:43:35,298 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 37eda6b0dad56caa0fa6c1ac8208d5b2 in 283ms, sequenceid=155, compaction requested=false 2024-11-07T12:43:35,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for 37eda6b0dad56caa0fa6c1ac8208d5b2: 2024-11-07T12:43:35,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:35,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-07T12:43:35,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-07T12:43:35,301 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-07T12:43:35,301 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 903 msec 2024-11-07T12:43:35,303 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 910 msec 2024-11-07T12:43:35,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:35,363 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 37eda6b0dad56caa0fa6c1ac8208d5b2 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-07T12:43:35,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=A 2024-11-07T12:43:35,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:35,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=B 2024-11-07T12:43:35,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:35,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=C 2024-11-07T12:43:35,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:35,371 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411070b73a5535b5e43779ccda789e039516b_37eda6b0dad56caa0fa6c1ac8208d5b2 is 50, key is test_row_0/A:col10/1730983415362/Put/seqid=0 2024-11-07T12:43:35,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742525_1701 (size=14794) 2024-11-07T12:43:35,385 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:35,390 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411070b73a5535b5e43779ccda789e039516b_37eda6b0dad56caa0fa6c1ac8208d5b2 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411070b73a5535b5e43779ccda789e039516b_37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:35,391 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/656d2e3dca7a4df2ac13fce49e35dc97, store: [table=TestAcidGuarantees family=A region=37eda6b0dad56caa0fa6c1ac8208d5b2] 2024-11-07T12:43:35,391 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/656d2e3dca7a4df2ac13fce49e35dc97 is 175, key is test_row_0/A:col10/1730983415362/Put/seqid=0 2024-11-07T12:43:35,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:35,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47926 deadline: 1730983475383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:35,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:35,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1730983475387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:35,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:35,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:35,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:35,393 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:35,393 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:35,393 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:35,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:35,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47962 deadline: 1730983475390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:35,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:35,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47946 deadline: 1730983475391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:35,397 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:35,397 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:35,397 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:35,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:35,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47984 deadline: 1730983475392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:35,398 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:35,398 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:35,398 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:35,398 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:35,398 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:35,398 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:35,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742526_1702 (size=39749) 2024-11-07T12:43:35,405 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=172, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/656d2e3dca7a4df2ac13fce49e35dc97 2024-11-07T12:43:35,415 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/9b717e3fbf5c4fa19a794e88ae498682 is 50, key is test_row_0/B:col10/1730983415362/Put/seqid=0 2024-11-07T12:43:35,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742527_1703 (size=12151) 2024-11-07T12:43:35,449 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/9b717e3fbf5c4fa19a794e88ae498682 2024-11-07T12:43:35,470 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/887547cba1734220851859342c3e8621 is 50, key is test_row_0/C:col10/1730983415362/Put/seqid=0 2024-11-07T12:43:35,486 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 37eda6b0dad56caa0fa6c1ac8208d5b2#A#compaction#592 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:43:35,487 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/60b7b3d60b3d41fa9914eaf7b178f683 is 175, key is test_row_0/A:col10/1730983413710/Put/seqid=0 2024-11-07T12:43:35,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:35,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1730983475502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:35,505 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:35,505 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:35,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:35,505 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:35,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47962 deadline: 1730983475502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:35,506 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:35,506 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:35,506 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:35,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:35,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47926 deadline: 1730983475503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:35,506 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:35,506 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:35,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:35,506 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:35,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47984 deadline: 1730983475503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:35,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:35,506 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:35,506 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:35,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47946 deadline: 1730983475503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:35,506 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:35,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:35,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:35,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:35,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742528_1704 (size=12151) 2024-11-07T12:43:35,508 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/887547cba1734220851859342c3e8621 2024-11-07T12:43:35,513 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/656d2e3dca7a4df2ac13fce49e35dc97 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/656d2e3dca7a4df2ac13fce49e35dc97 2024-11-07T12:43:35,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742529_1705 (size=31363) 2024-11-07T12:43:35,520 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/656d2e3dca7a4df2ac13fce49e35dc97, entries=200, sequenceid=172, filesize=38.8 K 2024-11-07T12:43:35,521 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/9b717e3fbf5c4fa19a794e88ae498682 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/9b717e3fbf5c4fa19a794e88ae498682 2024-11-07T12:43:35,526 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/9b717e3fbf5c4fa19a794e88ae498682, entries=150, sequenceid=172, filesize=11.9 K 2024-11-07T12:43:35,527 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/887547cba1734220851859342c3e8621 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/887547cba1734220851859342c3e8621 2024-11-07T12:43:35,531 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/887547cba1734220851859342c3e8621, entries=150, sequenceid=172, filesize=11.9 K 2024-11-07T12:43:35,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-07T12:43:35,545 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:43:35,546 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:43:35,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-11-07T12:43:35,548 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:43:35,550 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:43:35,550 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:43:35,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-07T12:43:35,554 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 37eda6b0dad56caa0fa6c1ac8208d5b2 in 191ms, sequenceid=172, compaction requested=true 2024-11-07T12:43:35,554 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 37eda6b0dad56caa0fa6c1ac8208d5b2: 2024-11-07T12:43:35,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 37eda6b0dad56caa0fa6c1ac8208d5b2:A, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:43:35,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:43:35,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 37eda6b0dad56caa0fa6c1ac8208d5b2:B, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:43:35,555 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-07T12:43:35,555 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 37eda6b0dad56caa0fa6c1ac8208d5b2:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:43:35,555 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-07T12:43:35,564 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/3a5151d403bd42c683df1863703c8832 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/3a5151d403bd42c683df1863703c8832 2024-11-07T12:43:35,587 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 37eda6b0dad56caa0fa6c1ac8208d5b2/C of 37eda6b0dad56caa0fa6c1ac8208d5b2 into 3a5151d403bd42c683df1863703c8832(size=12.1 K), total size for store is 35.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:43:35,587 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 37eda6b0dad56caa0fa6c1ac8208d5b2: 2024-11-07T12:43:35,587 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., storeName=37eda6b0dad56caa0fa6c1ac8208d5b2/C, priority=13, startTime=1730983414979; duration=0sec 2024-11-07T12:43:35,587 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-07T12:43:35,587 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 37eda6b0dad56caa0fa6c1ac8208d5b2:C 2024-11-07T12:43:35,587 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 3 compacting, 2 eligible, 16 blocking 2024-11-07T12:43:35,593 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-07T12:43:35,593 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-07T12:43:35,593 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. because compaction request was cancelled 2024-11-07T12:43:35,593 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 37eda6b0dad56caa0fa6c1ac8208d5b2:A 2024-11-07T12:43:35,593 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 37eda6b0dad56caa0fa6c1ac8208d5b2:C 2024-11-07T12:43:35,593 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:43:35,597 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:43:35,597 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 37eda6b0dad56caa0fa6c1ac8208d5b2/B is initiating minor compaction (all files) 2024-11-07T12:43:35,597 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 37eda6b0dad56caa0fa6c1ac8208d5b2/B in TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:35,597 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/d08906e3ea444fe6a662fc09b0c48b13, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/4cd28318f27c4b3893ca37805a6b557f, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/9b717e3fbf5c4fa19a794e88ae498682] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp, totalSize=35.9 K 2024-11-07T12:43:35,599 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting d08906e3ea444fe6a662fc09b0c48b13, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1730983413696 2024-11-07T12:43:35,600 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 4cd28318f27c4b3893ca37805a6b557f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1730983414380 2024-11-07T12:43:35,601 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b717e3fbf5c4fa19a794e88ae498682, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1730983415035 2024-11-07T12:43:35,609 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 37eda6b0dad56caa0fa6c1ac8208d5b2#B#compaction#601 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:43:35,610 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/ca875722e7484770be1d900edc1e68d0 is 50, key is test_row_0/B:col10/1730983415362/Put/seqid=0 2024-11-07T12:43:35,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742530_1706 (size=12561) 2024-11-07T12:43:35,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-07T12:43:35,667 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/ca875722e7484770be1d900edc1e68d0 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/ca875722e7484770be1d900edc1e68d0 2024-11-07T12:43:35,676 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 37eda6b0dad56caa0fa6c1ac8208d5b2/B of 37eda6b0dad56caa0fa6c1ac8208d5b2 into ca875722e7484770be1d900edc1e68d0(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:43:35,676 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 37eda6b0dad56caa0fa6c1ac8208d5b2: 2024-11-07T12:43:35,676 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., storeName=37eda6b0dad56caa0fa6c1ac8208d5b2/B, priority=13, startTime=1730983415554; duration=0sec 2024-11-07T12:43:35,676 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:43:35,676 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 37eda6b0dad56caa0fa6c1ac8208d5b2:B 2024-11-07T12:43:35,676 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:43:35,678 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:43:35,678 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 37eda6b0dad56caa0fa6c1ac8208d5b2/C is initiating minor compaction (all files) 2024-11-07T12:43:35,678 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 37eda6b0dad56caa0fa6c1ac8208d5b2/C in TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:35,678 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/3a5151d403bd42c683df1863703c8832, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/8a0a74a3b90544c99771294d7a658410, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/887547cba1734220851859342c3e8621] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp, totalSize=35.9 K 2024-11-07T12:43:35,678 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a5151d403bd42c683df1863703c8832, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1730983413696 2024-11-07T12:43:35,679 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a0a74a3b90544c99771294d7a658410, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1730983414380 2024-11-07T12:43:35,679 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 887547cba1734220851859342c3e8621, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1730983415035 2024-11-07T12:43:35,688 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 37eda6b0dad56caa0fa6c1ac8208d5b2#C#compaction#602 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:43:35,688 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/4c78918acfa64990b1f4fc92fb5eb712 is 50, key is test_row_0/C:col10/1730983415362/Put/seqid=0 2024-11-07T12:43:35,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742531_1707 (size=12561) 2024-11-07T12:43:35,703 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-07T12:43:35,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:35,704 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing 37eda6b0dad56caa0fa6c1ac8208d5b2 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-07T12:43:35,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=A 2024-11-07T12:43:35,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:35,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=B 2024-11-07T12:43:35,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:35,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=C 2024-11-07T12:43:35,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:35,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107d7bd9db3043f41809b4e46f44954c807_37eda6b0dad56caa0fa6c1ac8208d5b2 is 50, key is test_row_0/A:col10/1730983415390/Put/seqid=0 2024-11-07T12:43:35,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:35,715 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. as already flushing 2024-11-07T12:43:35,718 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/4c78918acfa64990b1f4fc92fb5eb712 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/4c78918acfa64990b1f4fc92fb5eb712 2024-11-07T12:43:35,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742532_1708 (size=12304) 2024-11-07T12:43:35,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:35,724 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 37eda6b0dad56caa0fa6c1ac8208d5b2/C of 37eda6b0dad56caa0fa6c1ac8208d5b2 into 4c78918acfa64990b1f4fc92fb5eb712(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:43:35,724 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 37eda6b0dad56caa0fa6c1ac8208d5b2: 2024-11-07T12:43:35,724 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., storeName=37eda6b0dad56caa0fa6c1ac8208d5b2/C, priority=13, startTime=1730983415555; duration=0sec 2024-11-07T12:43:35,724 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:35,724 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 37eda6b0dad56caa0fa6c1ac8208d5b2:C 2024-11-07T12:43:35,726 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107d7bd9db3043f41809b4e46f44954c807_37eda6b0dad56caa0fa6c1ac8208d5b2 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107d7bd9db3043f41809b4e46f44954c807_37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:35,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/14da15b5f9be4df6827c69aa5b237cb2, store: [table=TestAcidGuarantees family=A region=37eda6b0dad56caa0fa6c1ac8208d5b2] 2024-11-07T12:43:35,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/14da15b5f9be4df6827c69aa5b237cb2 is 175, key is test_row_0/A:col10/1730983415390/Put/seqid=0 2024-11-07T12:43:35,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742533_1709 (size=31105) 2024-11-07T12:43:35,730 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=190, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/14da15b5f9be4df6827c69aa5b237cb2 2024-11-07T12:43:35,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:35,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47962 deadline: 1730983475728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:35,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:35,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47984 deadline: 1730983475728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:35,735 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:35,735 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:35,735 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:35,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:35,735 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:35,735 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:35,735 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:35,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47926 deadline: 1730983475729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:35,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:35,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:35,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:35,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/23f837ded55744d5b6b513704f11a628 is 50, key is test_row_0/B:col10/1730983415390/Put/seqid=0 2024-11-07T12:43:35,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:35,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47946 deadline: 1730983475734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:35,739 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:35,739 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:35,739 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:35,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742534_1710 (size=12151) 2024-11-07T12:43:35,741 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/23f837ded55744d5b6b513704f11a628 2024-11-07T12:43:35,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:35,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1730983475735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:35,742 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:35,742 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:35,742 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:35,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/c5866565a00a41d8a527090f8406af1d is 50, key is test_row_0/C:col10/1730983415390/Put/seqid=0 2024-11-07T12:43:35,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742535_1711 (size=12151) 2024-11-07T12:43:35,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:35,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47962 deadline: 1730983475841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:35,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:35,846 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:35,846 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:35,846 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:35,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47984 deadline: 1730983475842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:35,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:35,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47926 deadline: 1730983475842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:35,846 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:35,846 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:35,846 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:35,846 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:35,846 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:35,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:35,846 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:35,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47946 deadline: 1730983475842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:35,847 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:35,847 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:35,847 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:35,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:35,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1730983475851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:35,855 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:35,855 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:35,855 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:35,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-07T12:43:35,920 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/60b7b3d60b3d41fa9914eaf7b178f683 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/60b7b3d60b3d41fa9914eaf7b178f683 2024-11-07T12:43:35,924 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 37eda6b0dad56caa0fa6c1ac8208d5b2/A of 37eda6b0dad56caa0fa6c1ac8208d5b2 into 60b7b3d60b3d41fa9914eaf7b178f683(size=30.6 K), total size for store is 99.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:43:35,925 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 37eda6b0dad56caa0fa6c1ac8208d5b2: 2024-11-07T12:43:35,925 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., storeName=37eda6b0dad56caa0fa6c1ac8208d5b2/A, priority=13, startTime=1730983414978; duration=0sec 2024-11-07T12:43:35,925 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:35,925 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 37eda6b0dad56caa0fa6c1ac8208d5b2:A 2024-11-07T12:43:36,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:36,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47926 deadline: 1730983476052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:36,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:36,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:36,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:36,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47962 deadline: 1730983476052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:36,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:36,057 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:36,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:36,057 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:36,057 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:36,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47984 deadline: 1730983476053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:36,057 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:36,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:36,057 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:36,057 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:36,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47946 deadline: 1730983476053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:36,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:36,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:36,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:36,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:36,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1730983476062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:36,064 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:36,064 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:36,064 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:36,154 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/c5866565a00a41d8a527090f8406af1d 2024-11-07T12:43:36,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/14da15b5f9be4df6827c69aa5b237cb2 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/14da15b5f9be4df6827c69aa5b237cb2 2024-11-07T12:43:36,161 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/14da15b5f9be4df6827c69aa5b237cb2, entries=150, sequenceid=190, filesize=30.4 K 2024-11-07T12:43:36,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/23f837ded55744d5b6b513704f11a628 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/23f837ded55744d5b6b513704f11a628 2024-11-07T12:43:36,165 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/23f837ded55744d5b6b513704f11a628, entries=150, sequenceid=190, filesize=11.9 K 2024-11-07T12:43:36,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/c5866565a00a41d8a527090f8406af1d as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/c5866565a00a41d8a527090f8406af1d 2024-11-07T12:43:36,169 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/c5866565a00a41d8a527090f8406af1d, entries=150, sequenceid=190, filesize=11.9 K 2024-11-07T12:43:36,170 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 37eda6b0dad56caa0fa6c1ac8208d5b2 in 467ms, sequenceid=190, compaction requested=true 2024-11-07T12:43:36,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for 37eda6b0dad56caa0fa6c1ac8208d5b2: 2024-11-07T12:43:36,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:36,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-11-07T12:43:36,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-11-07T12:43:36,174 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-07T12:43:36,174 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 622 msec 2024-11-07T12:43:36,175 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 628 msec 2024-11-07T12:43:36,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-07T12:43:36,182 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:43:36,182 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:43:36,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees 2024-11-07T12:43:36,184 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:43:36,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-07T12:43:36,184 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:43:36,184 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:43:36,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-07T12:43:36,337 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-07T12:43:36,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:36,337 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2902): Flushing 37eda6b0dad56caa0fa6c1ac8208d5b2 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-07T12:43:36,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=A 2024-11-07T12:43:36,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:36,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=B 2024-11-07T12:43:36,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:36,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=C 2024-11-07T12:43:36,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:36,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107698199540cbf46fc83057360807867fc_37eda6b0dad56caa0fa6c1ac8208d5b2 is 50, key is test_row_0/A:col10/1730983415732/Put/seqid=0 2024-11-07T12:43:36,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742536_1712 (size=12304) 2024-11-07T12:43:36,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:36,356 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107698199540cbf46fc83057360807867fc_37eda6b0dad56caa0fa6c1ac8208d5b2 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107698199540cbf46fc83057360807867fc_37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:36,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/40499930c9d44168bcb645af50ac9dfa, store: [table=TestAcidGuarantees family=A region=37eda6b0dad56caa0fa6c1ac8208d5b2] 2024-11-07T12:43:36,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/40499930c9d44168bcb645af50ac9dfa is 175, key is test_row_0/A:col10/1730983415732/Put/seqid=0 2024-11-07T12:43:36,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:36,366 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. as already flushing 2024-11-07T12:43:36,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742537_1713 (size=31105) 2024-11-07T12:43:36,382 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=209, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/40499930c9d44168bcb645af50ac9dfa 2024-11-07T12:43:36,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:36,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47946 deadline: 1730983476377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:36,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:36,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1730983476377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:36,384 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:36,385 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:36,385 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:36,385 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:36,385 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:36,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:36,385 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:36,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47926 deadline: 1730983476378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:36,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:36,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:36,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:36,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/fa1148d0606c4a80ab3e7ad925a34da9 is 50, key is test_row_0/B:col10/1730983415732/Put/seqid=0 2024-11-07T12:43:36,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:36,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47984 deadline: 1730983476383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:36,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:36,390 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:36,390 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:36,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47962 deadline: 1730983476384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:36,390 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:36,390 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:36,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:36,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:36,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742538_1714 (size=12151) 2024-11-07T12:43:36,404 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/fa1148d0606c4a80ab3e7ad925a34da9 2024-11-07T12:43:36,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/5633e60e0c514eebb7ff263aff810a31 is 50, key is test_row_0/C:col10/1730983415732/Put/seqid=0 2024-11-07T12:43:36,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742539_1715 (size=12151) 2024-11-07T12:43:36,412 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/5633e60e0c514eebb7ff263aff810a31 2024-11-07T12:43:36,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/40499930c9d44168bcb645af50ac9dfa as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/40499930c9d44168bcb645af50ac9dfa 2024-11-07T12:43:36,419 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/40499930c9d44168bcb645af50ac9dfa, entries=150, sequenceid=209, filesize=30.4 K 2024-11-07T12:43:36,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/fa1148d0606c4a80ab3e7ad925a34da9 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/fa1148d0606c4a80ab3e7ad925a34da9 2024-11-07T12:43:36,423 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/fa1148d0606c4a80ab3e7ad925a34da9, entries=150, sequenceid=209, filesize=11.9 K 2024-11-07T12:43:36,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/5633e60e0c514eebb7ff263aff810a31 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/5633e60e0c514eebb7ff263aff810a31 2024-11-07T12:43:36,428 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/5633e60e0c514eebb7ff263aff810a31, entries=150, sequenceid=209, filesize=11.9 K 2024-11-07T12:43:36,429 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 37eda6b0dad56caa0fa6c1ac8208d5b2 in 91ms, sequenceid=209, compaction requested=true 2024-11-07T12:43:36,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2603): Flush status journal for 37eda6b0dad56caa0fa6c1ac8208d5b2: 2024-11-07T12:43:36,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:36,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=167 2024-11-07T12:43:36,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4169): Remote procedure done, pid=167 2024-11-07T12:43:36,432 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-07T12:43:36,432 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 246 msec 2024-11-07T12:43:36,436 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees in 251 msec 2024-11-07T12:43:36,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:36,495 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 37eda6b0dad56caa0fa6c1ac8208d5b2 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-07T12:43:36,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=A 2024-11-07T12:43:36,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:36,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=B 2024-11-07T12:43:36,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:36,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=C 2024-11-07T12:43:36,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:36,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-07T12:43:36,502 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:43:36,502 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411070d9738eba2254ff8991182ff91c4e8cc_37eda6b0dad56caa0fa6c1ac8208d5b2 is 50, key is test_row_0/A:col10/1730983416381/Put/seqid=0 2024-11-07T12:43:36,502 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:43:36,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees 2024-11-07T12:43:36,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-07T12:43:36,504 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:43:36,505 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:43:36,505 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:43:36,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742540_1716 (size=14794) 2024-11-07T12:43:36,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:36,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47926 deadline: 1730983476510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:36,518 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:36,518 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:36,518 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:36,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:36,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1730983476513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:36,518 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:36,518 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:36,519 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:36,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:36,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47946 deadline: 1730983476516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:36,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:36,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:36,523 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:36,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:36,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47984 deadline: 1730983476517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:36,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:36,523 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:36,523 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:36,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47962 deadline: 1730983476518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:36,523 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:36,524 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:36,524 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:36,524 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:36,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-07T12:43:36,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:36,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47926 deadline: 1730983476622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:36,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:36,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1730983476622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:36,628 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:36,628 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:36,628 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:36,628 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:36,628 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:36,628 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:36,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:36,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47946 deadline: 1730983476631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:36,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:36,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:36,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:36,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:36,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47962 deadline: 1730983476631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:36,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:36,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:36,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:36,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:36,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47984 deadline: 1730983476631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:36,635 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:36,635 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:36,635 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:36,657 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-07T12:43:36,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:36,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. as already flushing 2024-11-07T12:43:36,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:36,658 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:36,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:36,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:36,811 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-07T12:43:36,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:36,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. as already flushing 2024-11-07T12:43:36,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:36,811 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:36,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:36,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:36,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-07T12:43:36,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:36,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47926 deadline: 1730983476831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:36,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:36,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:36,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:36,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1730983476832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:36,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:36,836 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:36,836 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:36,836 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:36,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:36,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47946 deadline: 1730983476842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:36,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:36,845 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:36,845 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:36,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47962 deadline: 1730983476842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:36,845 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:36,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:36,845 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:36,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47984 deadline: 1730983476842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:36,845 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:36,845 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:36,846 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:36,846 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:36,846 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:36,909 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:36,925 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411070d9738eba2254ff8991182ff91c4e8cc_37eda6b0dad56caa0fa6c1ac8208d5b2 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411070d9738eba2254ff8991182ff91c4e8cc_37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:36,927 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/b9bcc05e42fb438f84da4b0fc99cda02, store: [table=TestAcidGuarantees family=A region=37eda6b0dad56caa0fa6c1ac8208d5b2] 2024-11-07T12:43:36,928 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/b9bcc05e42fb438f84da4b0fc99cda02 is 175, key is test_row_0/A:col10/1730983416381/Put/seqid=0 2024-11-07T12:43:36,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742541_1717 (size=39749) 2024-11-07T12:43:36,953 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=225, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/b9bcc05e42fb438f84da4b0fc99cda02 2024-11-07T12:43:36,964 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-07T12:43:36,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:36,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. as already flushing 2024-11-07T12:43:36,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:36,965 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:36,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:36,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:36,990 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/c6eba27ff52c42778f7bfd90259b3051 is 50, key is test_row_0/B:col10/1730983416381/Put/seqid=0 2024-11-07T12:43:37,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742542_1718 (size=12151) 2024-11-07T12:43:37,004 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/c6eba27ff52c42778f7bfd90259b3051 2024-11-07T12:43:37,018 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/e048a892172b46f78a4c0f67a3fcb5a5 is 50, key is test_row_0/C:col10/1730983416381/Put/seqid=0 2024-11-07T12:43:37,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742543_1719 (size=12151) 2024-11-07T12:43:37,050 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/e048a892172b46f78a4c0f67a3fcb5a5 2024-11-07T12:43:37,055 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/b9bcc05e42fb438f84da4b0fc99cda02 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/b9bcc05e42fb438f84da4b0fc99cda02 2024-11-07T12:43:37,058 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/b9bcc05e42fb438f84da4b0fc99cda02, entries=200, sequenceid=225, filesize=38.8 K 2024-11-07T12:43:37,059 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/c6eba27ff52c42778f7bfd90259b3051 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/c6eba27ff52c42778f7bfd90259b3051 2024-11-07T12:43:37,063 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/c6eba27ff52c42778f7bfd90259b3051, entries=150, sequenceid=225, filesize=11.9 K 2024-11-07T12:43:37,063 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/e048a892172b46f78a4c0f67a3fcb5a5 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/e048a892172b46f78a4c0f67a3fcb5a5 2024-11-07T12:43:37,067 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/e048a892172b46f78a4c0f67a3fcb5a5, entries=150, sequenceid=225, filesize=11.9 K 2024-11-07T12:43:37,067 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 37eda6b0dad56caa0fa6c1ac8208d5b2 in 572ms, sequenceid=225, compaction requested=true 2024-11-07T12:43:37,067 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 37eda6b0dad56caa0fa6c1ac8208d5b2: 2024-11-07T12:43:37,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 37eda6b0dad56caa0fa6c1ac8208d5b2:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:43:37,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:37,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 37eda6b0dad56caa0fa6c1ac8208d5b2:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:43:37,068 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-07T12:43:37,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:37,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 37eda6b0dad56caa0fa6c1ac8208d5b2:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:43:37,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:43:37,068 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:43:37,069 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:43:37,069 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 37eda6b0dad56caa0fa6c1ac8208d5b2/B is initiating minor compaction (all files) 2024-11-07T12:43:37,069 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 37eda6b0dad56caa0fa6c1ac8208d5b2/B in TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:37,069 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/ca875722e7484770be1d900edc1e68d0, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/23f837ded55744d5b6b513704f11a628, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/fa1148d0606c4a80ab3e7ad925a34da9, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/c6eba27ff52c42778f7bfd90259b3051] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp, totalSize=47.9 K 2024-11-07T12:43:37,069 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 204176 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-07T12:43:37,069 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): 37eda6b0dad56caa0fa6c1ac8208d5b2/A is initiating minor compaction (all files) 2024-11-07T12:43:37,069 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 37eda6b0dad56caa0fa6c1ac8208d5b2/A in TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:37,069 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/60b7b3d60b3d41fa9914eaf7b178f683, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/ad853594617743f9b3f8ccc4d4633d23, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/656d2e3dca7a4df2ac13fce49e35dc97, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/14da15b5f9be4df6827c69aa5b237cb2, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/40499930c9d44168bcb645af50ac9dfa, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/b9bcc05e42fb438f84da4b0fc99cda02] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp, totalSize=199.4 K 2024-11-07T12:43:37,069 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting ca875722e7484770be1d900edc1e68d0, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1730983415035 2024-11-07T12:43:37,069 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=10 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:37,069 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. files: [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/60b7b3d60b3d41fa9914eaf7b178f683, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/ad853594617743f9b3f8ccc4d4633d23, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/656d2e3dca7a4df2ac13fce49e35dc97, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/14da15b5f9be4df6827c69aa5b237cb2, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/40499930c9d44168bcb645af50ac9dfa, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/b9bcc05e42fb438f84da4b0fc99cda02] 2024-11-07T12:43:37,070 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 23f837ded55744d5b6b513704f11a628, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1730983415378 2024-11-07T12:43:37,070 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60b7b3d60b3d41fa9914eaf7b178f683, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1730983413696 2024-11-07T12:43:37,070 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting fa1148d0606c4a80ab3e7ad925a34da9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1730983415722 2024-11-07T12:43:37,070 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad853594617743f9b3f8ccc4d4633d23, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1730983414380 2024-11-07T12:43:37,071 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting c6eba27ff52c42778f7bfd90259b3051, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1730983416381 2024-11-07T12:43:37,071 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 656d2e3dca7a4df2ac13fce49e35dc97, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1730983415035 2024-11-07T12:43:37,071 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14da15b5f9be4df6827c69aa5b237cb2, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1730983415378 2024-11-07T12:43:37,072 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40499930c9d44168bcb645af50ac9dfa, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1730983415722 2024-11-07T12:43:37,072 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9bcc05e42fb438f84da4b0fc99cda02, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1730983416375 2024-11-07T12:43:37,079 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 37eda6b0dad56caa0fa6c1ac8208d5b2#B#compaction#612 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:43:37,079 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/ce9dd77ac9334b91a5d159f6794939ca is 50, key is test_row_0/B:col10/1730983416381/Put/seqid=0 2024-11-07T12:43:37,081 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=37eda6b0dad56caa0fa6c1ac8208d5b2] 2024-11-07T12:43:37,101 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411076f9ed0bb927441eaa9364c8dd91ab92d_37eda6b0dad56caa0fa6c1ac8208d5b2 store=[table=TestAcidGuarantees family=A region=37eda6b0dad56caa0fa6c1ac8208d5b2] 2024-11-07T12:43:37,104 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411076f9ed0bb927441eaa9364c8dd91ab92d_37eda6b0dad56caa0fa6c1ac8208d5b2, store=[table=TestAcidGuarantees family=A region=37eda6b0dad56caa0fa6c1ac8208d5b2] 2024-11-07T12:43:37,104 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411076f9ed0bb927441eaa9364c8dd91ab92d_37eda6b0dad56caa0fa6c1ac8208d5b2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=37eda6b0dad56caa0fa6c1ac8208d5b2] 2024-11-07T12:43:37,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742544_1720 (size=12697) 2024-11-07T12:43:37,123 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-07T12:43:37,125 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/ce9dd77ac9334b91a5d159f6794939ca as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/ce9dd77ac9334b91a5d159f6794939ca 2024-11-07T12:43:37,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:37,129 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2902): Flushing 37eda6b0dad56caa0fa6c1ac8208d5b2 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-07T12:43:37,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=A 2024-11-07T12:43:37,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:37,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=B 2024-11-07T12:43:37,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:37,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=C 2024-11-07T12:43:37,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:37,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-07T12:43:37,133 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 37eda6b0dad56caa0fa6c1ac8208d5b2/B of 37eda6b0dad56caa0fa6c1ac8208d5b2 into ce9dd77ac9334b91a5d159f6794939ca(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:43:37,133 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 37eda6b0dad56caa0fa6c1ac8208d5b2: 2024-11-07T12:43:37,133 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., storeName=37eda6b0dad56caa0fa6c1ac8208d5b2/B, priority=12, startTime=1730983417068; duration=0sec 2024-11-07T12:43:37,133 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:43:37,133 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 37eda6b0dad56caa0fa6c1ac8208d5b2:B 2024-11-07T12:43:37,133 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:43:37,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742545_1721 (size=4469) 2024-11-07T12:43:37,136 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:43:37,136 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 37eda6b0dad56caa0fa6c1ac8208d5b2#A#compaction#613 average throughput is 0.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:43:37,136 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 37eda6b0dad56caa0fa6c1ac8208d5b2/C is initiating minor compaction (all files) 2024-11-07T12:43:37,136 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 37eda6b0dad56caa0fa6c1ac8208d5b2/C in TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:37,136 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/4c78918acfa64990b1f4fc92fb5eb712, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/c5866565a00a41d8a527090f8406af1d, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/5633e60e0c514eebb7ff263aff810a31, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/e048a892172b46f78a4c0f67a3fcb5a5] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp, totalSize=47.9 K 2024-11-07T12:43:37,136 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/c0a81589ffca484197b5a1c574e0132e is 175, key is test_row_0/A:col10/1730983416381/Put/seqid=0 2024-11-07T12:43:37,137 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c78918acfa64990b1f4fc92fb5eb712, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1730983415035 2024-11-07T12:43:37,138 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting c5866565a00a41d8a527090f8406af1d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1730983415378 2024-11-07T12:43:37,139 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 5633e60e0c514eebb7ff263aff810a31, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1730983415722 2024-11-07T12:43:37,140 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting e048a892172b46f78a4c0f67a3fcb5a5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1730983416381 2024-11-07T12:43:37,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:37,145 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. as already flushing 2024-11-07T12:43:37,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107483524e7f47f402085e1e26abd777a88_37eda6b0dad56caa0fa6c1ac8208d5b2 is 50, key is test_row_0/A:col10/1730983416516/Put/seqid=0 2024-11-07T12:43:37,161 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 37eda6b0dad56caa0fa6c1ac8208d5b2#C#compaction#615 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:43:37,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742546_1722 (size=31617) 2024-11-07T12:43:37,161 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/4168fddf4c3b4621bbe51525802b1b02 is 50, key is test_row_0/C:col10/1730983416381/Put/seqid=0 2024-11-07T12:43:37,170 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/c0a81589ffca484197b5a1c574e0132e as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/c0a81589ffca484197b5a1c574e0132e 2024-11-07T12:43:37,174 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 37eda6b0dad56caa0fa6c1ac8208d5b2/A of 37eda6b0dad56caa0fa6c1ac8208d5b2 into c0a81589ffca484197b5a1c574e0132e(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:43:37,174 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 37eda6b0dad56caa0fa6c1ac8208d5b2: 2024-11-07T12:43:37,174 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., storeName=37eda6b0dad56caa0fa6c1ac8208d5b2/A, priority=10, startTime=1730983417068; duration=0sec 2024-11-07T12:43:37,174 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:37,174 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 37eda6b0dad56caa0fa6c1ac8208d5b2:A 2024-11-07T12:43:37,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:37,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47926 deadline: 1730983477167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:37,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:37,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:37,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:37,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47984 deadline: 1730983477168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:37,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:37,177 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:37,177 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:37,177 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:37,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:37,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47962 deadline: 1730983477175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:37,179 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:37,179 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:37,179 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:37,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:37,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47946 deadline: 1730983477177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:37,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:37,183 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:37,183 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:37,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1730983477177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:37,183 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:37,183 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:37,183 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:37,183 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:37,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742548_1724 (size=12697) 2024-11-07T12:43:37,216 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/4168fddf4c3b4621bbe51525802b1b02 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/4168fddf4c3b4621bbe51525802b1b02 2024-11-07T12:43:37,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742547_1723 (size=12304) 2024-11-07T12:43:37,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:37,226 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 37eda6b0dad56caa0fa6c1ac8208d5b2/C of 37eda6b0dad56caa0fa6c1ac8208d5b2 into 4168fddf4c3b4621bbe51525802b1b02(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:43:37,226 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 37eda6b0dad56caa0fa6c1ac8208d5b2: 2024-11-07T12:43:37,226 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., storeName=37eda6b0dad56caa0fa6c1ac8208d5b2/C, priority=12, startTime=1730983417068; duration=0sec 2024-11-07T12:43:37,226 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:37,226 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 37eda6b0dad56caa0fa6c1ac8208d5b2:C 2024-11-07T12:43:37,228 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107483524e7f47f402085e1e26abd777a88_37eda6b0dad56caa0fa6c1ac8208d5b2 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107483524e7f47f402085e1e26abd777a88_37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:37,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/61c15cd61ec44297b2d8b03b23d0a19c, store: [table=TestAcidGuarantees family=A region=37eda6b0dad56caa0fa6c1ac8208d5b2] 2024-11-07T12:43:37,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/61c15cd61ec44297b2d8b03b23d0a19c is 175, key is test_row_0/A:col10/1730983416516/Put/seqid=0 2024-11-07T12:43:37,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742549_1725 (size=31105) 2024-11-07T12:43:37,235 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=241, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/61c15cd61ec44297b2d8b03b23d0a19c 2024-11-07T12:43:37,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/7e3b4c8e9d9b4fa18df7c5e298a5b99b is 50, key is test_row_0/B:col10/1730983416516/Put/seqid=0 2024-11-07T12:43:37,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742550_1726 (size=12151) 2024-11-07T12:43:37,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:37,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47926 deadline: 1730983477282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:37,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:37,284 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:37,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47984 deadline: 1730983477282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:37,284 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:37,284 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:37,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:37,284 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:37,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47962 deadline: 1730983477282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:37,285 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:37,285 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:37,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:37,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:37,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:37,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:37,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47946 deadline: 1730983477291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:37,295 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:37,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:37,295 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:37,295 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:37,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1730983477291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:37,296 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:37,296 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:37,296 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:37,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:37,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47926 deadline: 1730983477492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:37,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:37,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:37,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47984 deadline: 1730983477492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:37,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:37,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:37,498 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:37,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:37,498 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:37,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47962 deadline: 1730983477492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:37,498 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:37,498 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:37,498 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:37,498 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:37,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:37,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47946 deadline: 1730983477501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:37,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:37,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:37,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:37,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:37,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1730983477501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:37,505 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:37,505 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:37,505 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:37,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-07T12:43:37,658 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/7e3b4c8e9d9b4fa18df7c5e298a5b99b 2024-11-07T12:43:37,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/ac55478ae4e4458b9b2d359948466f77 is 50, key is test_row_0/C:col10/1730983416516/Put/seqid=0 2024-11-07T12:43:37,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742551_1727 (size=12151) 2024-11-07T12:43:37,668 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/ac55478ae4e4458b9b2d359948466f77 2024-11-07T12:43:37,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/61c15cd61ec44297b2d8b03b23d0a19c as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/61c15cd61ec44297b2d8b03b23d0a19c 2024-11-07T12:43:37,675 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/61c15cd61ec44297b2d8b03b23d0a19c, entries=150, sequenceid=241, filesize=30.4 K 2024-11-07T12:43:37,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/7e3b4c8e9d9b4fa18df7c5e298a5b99b as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/7e3b4c8e9d9b4fa18df7c5e298a5b99b 2024-11-07T12:43:37,680 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/7e3b4c8e9d9b4fa18df7c5e298a5b99b, entries=150, sequenceid=241, filesize=11.9 K 2024-11-07T12:43:37,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/ac55478ae4e4458b9b2d359948466f77 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/ac55478ae4e4458b9b2d359948466f77 2024-11-07T12:43:37,683 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/ac55478ae4e4458b9b2d359948466f77, entries=150, sequenceid=241, filesize=11.9 K 2024-11-07T12:43:37,684 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 37eda6b0dad56caa0fa6c1ac8208d5b2 in 556ms, sequenceid=241, compaction requested=false 2024-11-07T12:43:37,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2603): Flush status journal for 37eda6b0dad56caa0fa6c1ac8208d5b2: 2024-11-07T12:43:37,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:37,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=169 2024-11-07T12:43:37,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4169): Remote procedure done, pid=169 2024-11-07T12:43:37,688 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-11-07T12:43:37,688 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1810 sec 2024-11-07T12:43:37,690 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees in 1.1860 sec 2024-11-07T12:43:37,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:37,806 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 37eda6b0dad56caa0fa6c1ac8208d5b2 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-07T12:43:37,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=A 2024-11-07T12:43:37,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:37,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=B 2024-11-07T12:43:37,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:37,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=C 2024-11-07T12:43:37,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:37,822 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107d2da596e1bb64ab7beb5bd88937be3e9_37eda6b0dad56caa0fa6c1ac8208d5b2 is 50, key is test_row_0/A:col10/1730983417805/Put/seqid=0 2024-11-07T12:43:37,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742552_1728 (size=14944) 2024-11-07T12:43:37,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:37,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47962 deadline: 1730983477825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:37,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:37,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47984 deadline: 1730983477827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:37,836 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:37,836 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:37,836 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:37,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:37,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47946 deadline: 1730983477831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:37,840 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:37,840 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:37,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:37,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:37,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1730983477832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:37,841 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:37,841 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:37,841 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:37,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:37,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47926 deadline: 1730983477834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:37,841 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:37,841 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:37,841 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:37,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:37,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:37,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:37,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:37,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47984 deadline: 1730983477942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:37,946 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:37,946 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:37,946 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:37,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:37,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47962 deadline: 1730983477952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:37,954 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:37,954 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:37,954 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:37,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:37,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47946 deadline: 1730983477952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:37,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:37,958 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:37,958 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:37,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47926 deadline: 1730983477952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:37,958 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:37,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:37,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:37,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:37,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1730983477952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:37,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:37,958 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:37,959 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:37,959 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:38,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:38,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47984 deadline: 1730983478152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:38,160 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:38,160 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:38,160 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:38,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:38,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47946 deadline: 1730983478162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:38,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:38,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:38,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:38,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:38,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47962 deadline: 1730983478164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:38,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:38,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:38,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:38,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:38,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47926 deadline: 1730983478165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:38,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:38,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1730983478165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:38,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:38,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:38,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:38,167 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:38,167 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:38,167 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:38,231 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,235 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107d2da596e1bb64ab7beb5bd88937be3e9_37eda6b0dad56caa0fa6c1ac8208d5b2 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107d2da596e1bb64ab7beb5bd88937be3e9_37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:38,235 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/f48af124758a4aebb4b22549fe26f479, store: [table=TestAcidGuarantees family=A region=37eda6b0dad56caa0fa6c1ac8208d5b2] 2024-11-07T12:43:38,236 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/f48af124758a4aebb4b22549fe26f479 is 175, key is test_row_0/A:col10/1730983417805/Put/seqid=0 2024-11-07T12:43:38,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742553_1729 (size=39899) 2024-11-07T12:43:38,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:38,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47984 deadline: 1730983478462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:38,466 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:38,466 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:38,466 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:38,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:38,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47962 deadline: 1730983478471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:38,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:38,474 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:38,474 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:38,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47926 deadline: 1730983478471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:38,474 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:38,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:38,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47946 deadline: 1730983478472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:38,474 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:38,474 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:38,474 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:38,475 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:38,475 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:38,475 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:38,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:38,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1730983478472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:38,476 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:38,476 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:38,476 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:38,640 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=264, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/f48af124758a4aebb4b22549fe26f479 2024-11-07T12:43:38,649 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/a0adde41ab0a48b0bcae60c5b473efde is 50, key is test_row_0/B:col10/1730983417805/Put/seqid=0 2024-11-07T12:43:38,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-07T12:43:38,651 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:43:38,652 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:43:38,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] procedure2.ProcedureExecutor(1139): Stored pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees 2024-11-07T12:43:38,654 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:43:38,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-07T12:43:38,655 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:43:38,655 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:43:38,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742554_1730 (size=12251) 2024-11-07T12:43:38,659 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/a0adde41ab0a48b0bcae60c5b473efde 2024-11-07T12:43:38,667 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/5d1ddf827958462394b1e1fd3e323521 is 50, key is test_row_0/C:col10/1730983417805/Put/seqid=0 2024-11-07T12:43:38,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742555_1731 (size=12251) 2024-11-07T12:43:38,679 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/5d1ddf827958462394b1e1fd3e323521 2024-11-07T12:43:38,684 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/f48af124758a4aebb4b22549fe26f479 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/f48af124758a4aebb4b22549fe26f479 2024-11-07T12:43:38,687 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/f48af124758a4aebb4b22549fe26f479, entries=200, sequenceid=264, filesize=39.0 K 2024-11-07T12:43:38,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/a0adde41ab0a48b0bcae60c5b473efde as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/a0adde41ab0a48b0bcae60c5b473efde 2024-11-07T12:43:38,693 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/a0adde41ab0a48b0bcae60c5b473efde, entries=150, sequenceid=264, filesize=12.0 K 2024-11-07T12:43:38,694 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/5d1ddf827958462394b1e1fd3e323521 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/5d1ddf827958462394b1e1fd3e323521 2024-11-07T12:43:38,697 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/5d1ddf827958462394b1e1fd3e323521, entries=150, sequenceid=264, filesize=12.0 K 2024-11-07T12:43:38,698 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 37eda6b0dad56caa0fa6c1ac8208d5b2 in 892ms, sequenceid=264, compaction requested=true 2024-11-07T12:43:38,698 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 37eda6b0dad56caa0fa6c1ac8208d5b2: 2024-11-07T12:43:38,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 37eda6b0dad56caa0fa6c1ac8208d5b2:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:43:38,698 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:43:38,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:38,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 37eda6b0dad56caa0fa6c1ac8208d5b2:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:43:38,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:38,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 37eda6b0dad56caa0fa6c1ac8208d5b2:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:43:38,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:43:38,698 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:43:38,699 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102621 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:43:38,699 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): 37eda6b0dad56caa0fa6c1ac8208d5b2/A is initiating minor compaction (all files) 2024-11-07T12:43:38,699 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 37eda6b0dad56caa0fa6c1ac8208d5b2/A in TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:38,699 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/c0a81589ffca484197b5a1c574e0132e, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/61c15cd61ec44297b2d8b03b23d0a19c, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/f48af124758a4aebb4b22549fe26f479] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp, totalSize=100.2 K 2024-11-07T12:43:38,699 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:38,699 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. files: [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/c0a81589ffca484197b5a1c574e0132e, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/61c15cd61ec44297b2d8b03b23d0a19c, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/f48af124758a4aebb4b22549fe26f479] 2024-11-07T12:43:38,699 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37099 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:43:38,699 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0a81589ffca484197b5a1c574e0132e, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1730983416381 2024-11-07T12:43:38,699 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 37eda6b0dad56caa0fa6c1ac8208d5b2/B is initiating minor compaction (all files) 2024-11-07T12:43:38,700 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 37eda6b0dad56caa0fa6c1ac8208d5b2/B in TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:38,700 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/ce9dd77ac9334b91a5d159f6794939ca, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/7e3b4c8e9d9b4fa18df7c5e298a5b99b, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/a0adde41ab0a48b0bcae60c5b473efde] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp, totalSize=36.2 K 2024-11-07T12:43:38,700 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 61c15cd61ec44297b2d8b03b23d0a19c, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1730983416505 2024-11-07T12:43:38,701 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting ce9dd77ac9334b91a5d159f6794939ca, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1730983416381 2024-11-07T12:43:38,702 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting f48af124758a4aebb4b22549fe26f479, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1730983417172 2024-11-07T12:43:38,702 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e3b4c8e9d9b4fa18df7c5e298a5b99b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1730983416505 2024-11-07T12:43:38,702 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting a0adde41ab0a48b0bcae60c5b473efde, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1730983417172 2024-11-07T12:43:38,710 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=37eda6b0dad56caa0fa6c1ac8208d5b2] 2024-11-07T12:43:38,714 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 37eda6b0dad56caa0fa6c1ac8208d5b2#B#compaction#622 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:43:38,714 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/2aa93a34157a413c844ee47a6d32c7c4 is 50, key is test_row_0/B:col10/1730983417805/Put/seqid=0 2024-11-07T12:43:38,717 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024110741396e4ac05e4251a8b890688a7507d4_37eda6b0dad56caa0fa6c1ac8208d5b2 store=[table=TestAcidGuarantees family=A region=37eda6b0dad56caa0fa6c1ac8208d5b2] 2024-11-07T12:43:38,719 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024110741396e4ac05e4251a8b890688a7507d4_37eda6b0dad56caa0fa6c1ac8208d5b2, store=[table=TestAcidGuarantees family=A region=37eda6b0dad56caa0fa6c1ac8208d5b2] 2024-11-07T12:43:38,719 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110741396e4ac05e4251a8b890688a7507d4_37eda6b0dad56caa0fa6c1ac8208d5b2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=37eda6b0dad56caa0fa6c1ac8208d5b2] 2024-11-07T12:43:38,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742556_1732 (size=12899) 2024-11-07T12:43:38,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742557_1733 (size=4469) 2024-11-07T12:43:38,744 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/2aa93a34157a413c844ee47a6d32c7c4 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/2aa93a34157a413c844ee47a6d32c7c4 2024-11-07T12:43:38,749 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 37eda6b0dad56caa0fa6c1ac8208d5b2/B of 37eda6b0dad56caa0fa6c1ac8208d5b2 into 2aa93a34157a413c844ee47a6d32c7c4(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:43:38,749 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 37eda6b0dad56caa0fa6c1ac8208d5b2: 2024-11-07T12:43:38,749 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., storeName=37eda6b0dad56caa0fa6c1ac8208d5b2/B, priority=13, startTime=1730983418698; duration=0sec 2024-11-07T12:43:38,749 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:43:38,750 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 37eda6b0dad56caa0fa6c1ac8208d5b2:B 2024-11-07T12:43:38,750 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:43:38,755 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37099 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:43:38,756 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 37eda6b0dad56caa0fa6c1ac8208d5b2/C is initiating minor compaction (all files) 2024-11-07T12:43:38,756 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 37eda6b0dad56caa0fa6c1ac8208d5b2/C in TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:38,756 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/4168fddf4c3b4621bbe51525802b1b02, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/ac55478ae4e4458b9b2d359948466f77, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/5d1ddf827958462394b1e1fd3e323521] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp, totalSize=36.2 K 2024-11-07T12:43:38,756 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 4168fddf4c3b4621bbe51525802b1b02, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1730983416381 2024-11-07T12:43:38,756 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting ac55478ae4e4458b9b2d359948466f77, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1730983416505 2024-11-07T12:43:38,757 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d1ddf827958462394b1e1fd3e323521, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1730983417172 2024-11-07T12:43:38,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-07T12:43:38,767 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 37eda6b0dad56caa0fa6c1ac8208d5b2#C#compaction#623 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:43:38,767 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/ee64f4b0d8784f1db37c13caab95fff4 is 50, key is test_row_0/C:col10/1730983417805/Put/seqid=0 2024-11-07T12:43:38,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742558_1734 (size=12899) 2024-11-07T12:43:38,790 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/ee64f4b0d8784f1db37c13caab95fff4 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/ee64f4b0d8784f1db37c13caab95fff4 2024-11-07T12:43:38,795 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 37eda6b0dad56caa0fa6c1ac8208d5b2/C of 37eda6b0dad56caa0fa6c1ac8208d5b2 into ee64f4b0d8784f1db37c13caab95fff4(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:43:38,795 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 37eda6b0dad56caa0fa6c1ac8208d5b2: 2024-11-07T12:43:38,795 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., storeName=37eda6b0dad56caa0fa6c1ac8208d5b2/C, priority=13, startTime=1730983418698; duration=0sec 2024-11-07T12:43:38,795 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:38,795 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 37eda6b0dad56caa0fa6c1ac8208d5b2:C 2024-11-07T12:43:38,808 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-07T12:43:38,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:38,808 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2902): Flushing 37eda6b0dad56caa0fa6c1ac8208d5b2 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-07T12:43:38,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=A 2024-11-07T12:43:38,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:38,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=B 2024-11-07T12:43:38,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:38,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=C 2024-11-07T12:43:38,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:38,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107b9c10c8b522345fb8dfc9be947af040b_37eda6b0dad56caa0fa6c1ac8208d5b2 is 50, key is test_row_0/A:col10/1730983417826/Put/seqid=0 2024-11-07T12:43:38,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742559_1735 (size=12454) 2024-11-07T12:43:38,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,861 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107b9c10c8b522345fb8dfc9be947af040b_37eda6b0dad56caa0fa6c1ac8208d5b2 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107b9c10c8b522345fb8dfc9be947af040b_37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:38,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/cd3140b4967e46e88686729c0b59919f, store: [table=TestAcidGuarantees family=A region=37eda6b0dad56caa0fa6c1ac8208d5b2] 2024-11-07T12:43:38,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/cd3140b4967e46e88686729c0b59919f is 175, key is test_row_0/A:col10/1730983417826/Put/seqid=0 2024-11-07T12:43:38,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742560_1736 (size=31255) 2024-11-07T12:43:38,883 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=279, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/cd3140b4967e46e88686729c0b59919f 2024-11-07T12:43:38,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/be49a87cc52647efb51217fce97f85c3 is 50, key is test_row_0/B:col10/1730983417826/Put/seqid=0 2024-11-07T12:43:38,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742561_1737 (size=12301) 2024-11-07T12:43:38,906 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/be49a87cc52647efb51217fce97f85c3 2024-11-07T12:43:38,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/4e39228f309b4a119ce3fda53056ec5a is 50, key is test_row_0/C:col10/1730983417826/Put/seqid=0 2024-11-07T12:43:38,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742562_1738 (size=12301) 2024-11-07T12:43:38,930 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/4e39228f309b4a119ce3fda53056ec5a 2024-11-07T12:43:38,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/cd3140b4967e46e88686729c0b59919f as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/cd3140b4967e46e88686729c0b59919f 2024-11-07T12:43:38,939 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/cd3140b4967e46e88686729c0b59919f, entries=150, sequenceid=279, filesize=30.5 K 2024-11-07T12:43:38,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/be49a87cc52647efb51217fce97f85c3 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/be49a87cc52647efb51217fce97f85c3 2024-11-07T12:43:38,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,944 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/be49a87cc52647efb51217fce97f85c3, entries=150, sequenceid=279, filesize=12.0 K 2024-11-07T12:43:38,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/4e39228f309b4a119ce3fda53056ec5a as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/4e39228f309b4a119ce3fda53056ec5a 2024-11-07T12:43:38,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,951 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/4e39228f309b4a119ce3fda53056ec5a, entries=150, sequenceid=279, filesize=12.0 K 2024-11-07T12:43:38,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,952 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=0 B/0 for 37eda6b0dad56caa0fa6c1ac8208d5b2 in 144ms, sequenceid=279, compaction requested=false 2024-11-07T12:43:38,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2603): Flush status journal for 37eda6b0dad56caa0fa6c1ac8208d5b2: 2024-11-07T12:43:38,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:38,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=171 2024-11-07T12:43:38,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4169): Remote procedure done, pid=171 2024-11-07T12:43:38,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,955 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=171, resume processing ppid=170 2024-11-07T12:43:38,955 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, ppid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 298 msec 2024-11-07T12:43:38,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,957 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees in 304 msec 2024-11-07T12:43:38,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-07T12:43:38,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,972 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:43:38,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,973 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:43:38,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] procedure2.ProcedureExecutor(1139): Stored pid=172, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees 2024-11-07T12:43:38,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,977 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=172, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:43:38,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,978 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=172, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:43:38,978 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=173, ppid=172, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:43:38,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-07T12:43:38,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:38,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:39,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:39,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:39,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:39,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:39,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:39,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:39,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:39,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:39,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:39,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:39,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:39,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:39,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:39,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:39,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:39,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:39,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:39,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:39,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:39,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:39,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:39,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:39,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:39,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:39,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:39,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:39,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:39,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:39,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:39,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:39,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:39,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:39,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:39,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:39,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:39,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:39,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:39,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:39,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:39,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:39,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:39,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:39,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:39,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:39,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:39,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:39,021 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 37eda6b0dad56caa0fa6c1ac8208d5b2 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-07T12:43:39,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=A 2024-11-07T12:43:39,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:39,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=B 2024-11-07T12:43:39,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:39,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=C 2024-11-07T12:43:39,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:39,033 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411073b53cb1286dd4fb786b3898ff7619f77_37eda6b0dad56caa0fa6c1ac8208d5b2 is 50, key is test_row_0/A:col10/1730983419011/Put/seqid=0 2024-11-07T12:43:39,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742563_1739 (size=12454) 2024-11-07T12:43:39,079 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1730983419065, max timestamp=1730983419077Num uniques 859; 2024-11-07T12:43:39,079 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1730983419065, max timestamp=1730983419077Num uniques 859; 2024-11-07T12:43:39,079 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-07T12:43:39,079 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-07T12:43:39,081 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1730983419065, max timestamp=1730983419077Num uniques 859; 2024-11-07T12:43:39,081 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-07T12:43:39,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-07T12:43:39,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:39,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1730983479119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:39,127 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:39,127 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:39,127 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:39,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:39,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47962 deadline: 1730983479121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:39,128 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:39,129 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:39,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:39,129 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:39,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47926 deadline: 1730983479121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:39,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:39,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:39,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:39,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:39,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47946 deadline: 1730983479122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:39,129 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:39,129 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:39,129 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:39,131 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-07T12:43:39,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:39,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. as already flushing 2024-11-07T12:43:39,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:39,131 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:39,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:39,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:39,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:39,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47984 deadline: 1730983479127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:39,133 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:39,133 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:39,133 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:39,142 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 37eda6b0dad56caa0fa6c1ac8208d5b2#A#compaction#621 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:43:39,143 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/7fac4daa648d4ac9be32b290b192a61c is 175, key is test_row_0/A:col10/1730983417805/Put/seqid=0 2024-11-07T12:43:39,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742564_1740 (size=31819) 2024-11-07T12:43:39,152 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/7fac4daa648d4ac9be32b290b192a61c as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/7fac4daa648d4ac9be32b290b192a61c 2024-11-07T12:43:39,157 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 37eda6b0dad56caa0fa6c1ac8208d5b2/A of 37eda6b0dad56caa0fa6c1ac8208d5b2 into 7fac4daa648d4ac9be32b290b192a61c(size=31.1 K), total size for store is 61.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:43:39,157 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 37eda6b0dad56caa0fa6c1ac8208d5b2: 2024-11-07T12:43:39,157 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., storeName=37eda6b0dad56caa0fa6c1ac8208d5b2/A, priority=13, startTime=1730983418698; duration=0sec 2024-11-07T12:43:39,157 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:39,157 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 37eda6b0dad56caa0fa6c1ac8208d5b2:A 2024-11-07T12:43:39,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:39,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1730983479232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:39,234 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:39,234 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:39,234 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:39,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47962 deadline: 1730983479232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:39,234 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:39,234 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:39,234 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:39,234 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:39,234 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:39,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47926 deadline: 1730983479232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:39,234 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:39,234 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:39,235 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:39,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47946 deadline: 1730983479232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:39,235 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:39,235 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:39,235 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:39,235 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:39,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:39,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47984 deadline: 1730983479241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:39,246 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:39,246 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:39,246 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:39,283 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-07T12:43:39,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:39,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. as already flushing 2024-11-07T12:43:39,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:39,284 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:39,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:39,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:39,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-07T12:43:39,437 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-07T12:43:39,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:39,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. as already flushing 2024-11-07T12:43:39,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:39,437 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:39,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:39,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:39,443 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:39,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:39,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47962 deadline: 1730983479442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:39,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:39,446 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:39,446 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:39,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47926 deadline: 1730983479443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:39,446 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:39,447 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411073b53cb1286dd4fb786b3898ff7619f77_37eda6b0dad56caa0fa6c1ac8208d5b2 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411073b53cb1286dd4fb786b3898ff7619f77_37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:39,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:39,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:39,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:39,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:39,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47946 deadline: 1730983479443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:39,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:39,447 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:39,447 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:39,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1730983479443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:39,447 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:39,447 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/07bbeefee46c4ce49cfa3f62b0f24b77, store: [table=TestAcidGuarantees family=A region=37eda6b0dad56caa0fa6c1ac8208d5b2] 2024-11-07T12:43:39,448 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:39,448 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:39,448 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:39,448 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/07bbeefee46c4ce49cfa3f62b0f24b77 is 175, key is test_row_0/A:col10/1730983419011/Put/seqid=0 2024-11-07T12:43:39,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:39,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47984 deadline: 1730983479452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:39,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742565_1741 (size=31255) 2024-11-07T12:43:39,458 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:39,459 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:39,459 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:39,459 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=289, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/07bbeefee46c4ce49cfa3f62b0f24b77 2024-11-07T12:43:39,465 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/4cb385971a07498681662ecfecde0850 is 50, key is test_row_0/B:col10/1730983419011/Put/seqid=0 2024-11-07T12:43:39,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742566_1742 (size=12301) 2024-11-07T12:43:39,470 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/4cb385971a07498681662ecfecde0850 2024-11-07T12:43:39,476 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/607b0632f6ea4030b233ea5ba3c760be is 50, key is test_row_0/C:col10/1730983419011/Put/seqid=0 2024-11-07T12:43:39,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742567_1743 (size=12301) 2024-11-07T12:43:39,591 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-07T12:43:39,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:39,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. as already flushing 2024-11-07T12:43:39,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:39,591 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:39,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:39,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:39,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-07T12:43:39,744 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-07T12:43:39,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:39,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. as already flushing 2024-11-07T12:43:39,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:39,744 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:39,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:39,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:39,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:39,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47946 deadline: 1730983479751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:39,755 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:39,755 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:39,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47962 deadline: 1730983479751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:39,755 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:39,755 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:39,755 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:39,755 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:39,755 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:39,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47926 deadline: 1730983479751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:39,755 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:39,755 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:39,755 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:39,755 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:39,756 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:39,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1730983479751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:39,756 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:39,756 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:39,756 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:39,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:39,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47984 deadline: 1730983479772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:39,774 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:39,774 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:39,774 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:39,886 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/607b0632f6ea4030b233ea5ba3c760be 2024-11-07T12:43:39,890 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/07bbeefee46c4ce49cfa3f62b0f24b77 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/07bbeefee46c4ce49cfa3f62b0f24b77 2024-11-07T12:43:39,893 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/07bbeefee46c4ce49cfa3f62b0f24b77, entries=150, sequenceid=289, filesize=30.5 K 2024-11-07T12:43:39,894 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/4cb385971a07498681662ecfecde0850 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/4cb385971a07498681662ecfecde0850 2024-11-07T12:43:39,897 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/4cb385971a07498681662ecfecde0850, entries=150, sequenceid=289, filesize=12.0 K 2024-11-07T12:43:39,897 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-07T12:43:39,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:39,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. as already flushing 2024-11-07T12:43:39,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:39,897 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:39,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:39,898 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/607b0632f6ea4030b233ea5ba3c760be as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/607b0632f6ea4030b233ea5ba3c760be 2024-11-07T12:43:39,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:39,903 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/607b0632f6ea4030b233ea5ba3c760be, entries=150, sequenceid=289, filesize=12.0 K 2024-11-07T12:43:39,903 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=136.02 KB/139285 for 37eda6b0dad56caa0fa6c1ac8208d5b2 in 882ms, sequenceid=289, compaction requested=true 2024-11-07T12:43:39,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 37eda6b0dad56caa0fa6c1ac8208d5b2: 2024-11-07T12:43:39,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 37eda6b0dad56caa0fa6c1ac8208d5b2:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:43:39,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:39,904 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:43:39,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 37eda6b0dad56caa0fa6c1ac8208d5b2:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:43:39,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:39,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 37eda6b0dad56caa0fa6c1ac8208d5b2:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:43:39,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:43:39,904 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:43:39,906 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94329 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:43:39,906 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37501 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:43:39,906 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): 37eda6b0dad56caa0fa6c1ac8208d5b2/A is initiating minor compaction (all files) 2024-11-07T12:43:39,906 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 37eda6b0dad56caa0fa6c1ac8208d5b2/B is initiating minor compaction (all files) 2024-11-07T12:43:39,906 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 37eda6b0dad56caa0fa6c1ac8208d5b2/A in TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:39,906 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 37eda6b0dad56caa0fa6c1ac8208d5b2/B in TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:39,906 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/7fac4daa648d4ac9be32b290b192a61c, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/cd3140b4967e46e88686729c0b59919f, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/07bbeefee46c4ce49cfa3f62b0f24b77] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp, totalSize=92.1 K 2024-11-07T12:43:39,906 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:39,906 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/2aa93a34157a413c844ee47a6d32c7c4, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/be49a87cc52647efb51217fce97f85c3, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/4cb385971a07498681662ecfecde0850] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp, totalSize=36.6 K 2024-11-07T12:43:39,906 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. files: [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/7fac4daa648d4ac9be32b290b192a61c, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/cd3140b4967e46e88686729c0b59919f, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/07bbeefee46c4ce49cfa3f62b0f24b77] 2024-11-07T12:43:39,906 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7fac4daa648d4ac9be32b290b192a61c, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1730983417172 2024-11-07T12:43:39,906 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 2aa93a34157a413c844ee47a6d32c7c4, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1730983417172 2024-11-07T12:43:39,906 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd3140b4967e46e88686729c0b59919f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1730983417825 2024-11-07T12:43:39,906 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting be49a87cc52647efb51217fce97f85c3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1730983417825 2024-11-07T12:43:39,907 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 4cb385971a07498681662ecfecde0850, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1730983419011 2024-11-07T12:43:39,907 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07bbeefee46c4ce49cfa3f62b0f24b77, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1730983419011 2024-11-07T12:43:39,912 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=37eda6b0dad56caa0fa6c1ac8208d5b2] 2024-11-07T12:43:39,914 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 37eda6b0dad56caa0fa6c1ac8208d5b2#B#compaction#630 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:43:39,914 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/011fa10c6f0f49c6a487f835a1ba6702 is 50, key is test_row_0/B:col10/1730983419011/Put/seqid=0 2024-11-07T12:43:39,917 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024110754ef0cb684da493db6f2514167693d91_37eda6b0dad56caa0fa6c1ac8208d5b2 store=[table=TestAcidGuarantees family=A region=37eda6b0dad56caa0fa6c1ac8208d5b2] 2024-11-07T12:43:39,919 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024110754ef0cb684da493db6f2514167693d91_37eda6b0dad56caa0fa6c1ac8208d5b2, store=[table=TestAcidGuarantees family=A region=37eda6b0dad56caa0fa6c1ac8208d5b2] 2024-11-07T12:43:39,919 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110754ef0cb684da493db6f2514167693d91_37eda6b0dad56caa0fa6c1ac8208d5b2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=37eda6b0dad56caa0fa6c1ac8208d5b2] 2024-11-07T12:43:39,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742568_1744 (size=13051) 2024-11-07T12:43:39,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742569_1745 (size=4469) 2024-11-07T12:43:39,931 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/011fa10c6f0f49c6a487f835a1ba6702 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/011fa10c6f0f49c6a487f835a1ba6702 2024-11-07T12:43:39,932 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 37eda6b0dad56caa0fa6c1ac8208d5b2#A#compaction#631 average throughput is 1.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:43:39,932 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/aad8221e321945fe85605d38843ebb2f is 175, key is test_row_0/A:col10/1730983419011/Put/seqid=0 2024-11-07T12:43:39,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742570_1746 (size=31971) 2024-11-07T12:43:39,937 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 37eda6b0dad56caa0fa6c1ac8208d5b2/B of 37eda6b0dad56caa0fa6c1ac8208d5b2 into 011fa10c6f0f49c6a487f835a1ba6702(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:43:39,937 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 37eda6b0dad56caa0fa6c1ac8208d5b2: 2024-11-07T12:43:39,937 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., storeName=37eda6b0dad56caa0fa6c1ac8208d5b2/B, priority=13, startTime=1730983419904; duration=0sec 2024-11-07T12:43:39,937 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:43:39,937 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 37eda6b0dad56caa0fa6c1ac8208d5b2:B 2024-11-07T12:43:39,937 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:43:39,938 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37501 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:43:39,938 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 37eda6b0dad56caa0fa6c1ac8208d5b2/C is initiating minor compaction (all files) 2024-11-07T12:43:39,938 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 37eda6b0dad56caa0fa6c1ac8208d5b2/C in TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:39,938 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/ee64f4b0d8784f1db37c13caab95fff4, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/4e39228f309b4a119ce3fda53056ec5a, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/607b0632f6ea4030b233ea5ba3c760be] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp, totalSize=36.6 K 2024-11-07T12:43:39,940 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting ee64f4b0d8784f1db37c13caab95fff4, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1730983417172 2024-11-07T12:43:39,940 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e39228f309b4a119ce3fda53056ec5a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1730983417825 2024-11-07T12:43:39,940 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 607b0632f6ea4030b233ea5ba3c760be, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1730983419011 2024-11-07T12:43:39,944 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/aad8221e321945fe85605d38843ebb2f as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/aad8221e321945fe85605d38843ebb2f 2024-11-07T12:43:39,948 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 37eda6b0dad56caa0fa6c1ac8208d5b2#C#compaction#632 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:43:39,949 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/a1d98a94ed454b478ba1dd30c9c87505 is 50, key is test_row_0/C:col10/1730983419011/Put/seqid=0 2024-11-07T12:43:39,950 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 37eda6b0dad56caa0fa6c1ac8208d5b2/A of 37eda6b0dad56caa0fa6c1ac8208d5b2 into aad8221e321945fe85605d38843ebb2f(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:43:39,950 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 37eda6b0dad56caa0fa6c1ac8208d5b2: 2024-11-07T12:43:39,950 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., storeName=37eda6b0dad56caa0fa6c1ac8208d5b2/A, priority=13, startTime=1730983419904; duration=0sec 2024-11-07T12:43:39,950 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:39,950 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 37eda6b0dad56caa0fa6c1ac8208d5b2:A 2024-11-07T12:43:39,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742571_1747 (size=13051) 2024-11-07T12:43:40,052 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-07T12:43:40,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:40,052 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2902): Flushing 37eda6b0dad56caa0fa6c1ac8208d5b2 3/3 column families, dataSize=136.02 KB heapSize=384.46 KB 2024-11-07T12:43:40,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=A 2024-11-07T12:43:40,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:43:40,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=B 2024-11-07T12:43:40,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:43:40,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=C 2024-11-07T12:43:40,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:43:40,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411070e79acc8681c417885eacbd1d7171735_37eda6b0dad56caa0fa6c1ac8208d5b2 is 50, key is test_row_0/A:col10/1730983419125/Put/seqid=0 2024-11-07T12:43:40,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742572_1748 (size=12454) 2024-11-07T12:43:40,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:40,068 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411070e79acc8681c417885eacbd1d7171735_37eda6b0dad56caa0fa6c1ac8208d5b2 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411070e79acc8681c417885eacbd1d7171735_37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:40,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/558d96c6f1314fa085eb857e0bd46a64, store: [table=TestAcidGuarantees family=A region=37eda6b0dad56caa0fa6c1ac8208d5b2] 2024-11-07T12:43:40,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/558d96c6f1314fa085eb857e0bd46a64 is 175, key is test_row_0/A:col10/1730983419125/Put/seqid=0 2024-11-07T12:43:40,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742573_1749 (size=31255) 2024-11-07T12:43:40,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-07T12:43:40,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:40,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. as already flushing 2024-11-07T12:43:40,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:40,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47962 deadline: 1730983480267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:40,273 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:40,273 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:40,273 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:40,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:40,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47946 deadline: 1730983480271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:40,278 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:40,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:40,278 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:40,278 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:40,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47926 deadline: 1730983480273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:40,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:40,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:40,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:40,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1730983480273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:40,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:40,279 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:40,279 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:40,279 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:40,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:40,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47984 deadline: 1730983480282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:40,284 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:40,284 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:40,284 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:40,365 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/a1d98a94ed454b478ba1dd30c9c87505 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/a1d98a94ed454b478ba1dd30c9c87505 2024-11-07T12:43:40,369 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 37eda6b0dad56caa0fa6c1ac8208d5b2/C of 37eda6b0dad56caa0fa6c1ac8208d5b2 into a1d98a94ed454b478ba1dd30c9c87505(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:43:40,369 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 37eda6b0dad56caa0fa6c1ac8208d5b2: 2024-11-07T12:43:40,369 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., storeName=37eda6b0dad56caa0fa6c1ac8208d5b2/C, priority=13, startTime=1730983419904; duration=0sec 2024-11-07T12:43:40,369 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:40,369 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 37eda6b0dad56caa0fa6c1ac8208d5b2:C 2024-11-07T12:43:40,385 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:40,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:40,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47962 deadline: 1730983480382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:40,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47946 deadline: 1730983480382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:40,386 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:40,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:40,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:40,386 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:40,386 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:40,386 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:40,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47926 deadline: 1730983480382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:40,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1730983480382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:40,386 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:40,386 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:40,386 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:40,386 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:40,386 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:40,386 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:40,386 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:40,386 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:40,474 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=328, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/558d96c6f1314fa085eb857e0bd46a64 2024-11-07T12:43:40,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/8963df44fc5f42adaea6d1f1226e8eef is 50, key is test_row_0/B:col10/1730983419125/Put/seqid=0 2024-11-07T12:43:40,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742574_1750 (size=12301) 2024-11-07T12:43:40,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:40,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47962 deadline: 1730983480592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:40,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:40,595 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:40,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47926 deadline: 1730983480592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:40,595 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:40,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:40,595 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:40,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47946 deadline: 1730983480592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:40,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:40,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:40,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1730983480592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:40,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:40,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:40,595 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:40,595 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:40,595 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:40,595 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:40,595 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:40,595 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:40,890 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/8963df44fc5f42adaea6d1f1226e8eef 2024-11-07T12:43:40,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/e76b8ceadf3d4ab6a5f473d9d40210cf is 50, key is test_row_0/C:col10/1730983419125/Put/seqid=0 2024-11-07T12:43:40,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742575_1751 (size=12301) 2024-11-07T12:43:40,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:40,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47962 deadline: 1730983480901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:40,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:40,903 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:40,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47946 deadline: 1730983480902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:40,903 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:40,903 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:40,903 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:40,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:40,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:40,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:40,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1730983480902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:40,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:40,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47926 deadline: 1730983480902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:40,906 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:40,906 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:40,906 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:40,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:40,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:40,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:41,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-07T12:43:41,301 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.09 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/e76b8ceadf3d4ab6a5f473d9d40210cf 2024-11-07T12:43:41,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:41,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47984 deadline: 1730983481302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:41,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/558d96c6f1314fa085eb857e0bd46a64 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/558d96c6f1314fa085eb857e0bd46a64 2024-11-07T12:43:41,306 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to db9ad1cb6cf9,34293,1730983260217 for 'test_row_1' in 37eda6b0dad56caa0fa6c1ac8208d5b2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:43:41,306 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:41,306 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:41,306 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:41,309 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/558d96c6f1314fa085eb857e0bd46a64, entries=150, sequenceid=328, filesize=30.5 K 2024-11-07T12:43:41,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/8963df44fc5f42adaea6d1f1226e8eef as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/8963df44fc5f42adaea6d1f1226e8eef 2024-11-07T12:43:41,312 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/8963df44fc5f42adaea6d1f1226e8eef, entries=150, sequenceid=328, filesize=12.0 K 2024-11-07T12:43:41,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/e76b8ceadf3d4ab6a5f473d9d40210cf as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/e76b8ceadf3d4ab6a5f473d9d40210cf 2024-11-07T12:43:41,327 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/e76b8ceadf3d4ab6a5f473d9d40210cf, entries=150, sequenceid=328, filesize=12.0 K 2024-11-07T12:43:41,328 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(3140): Finished flush of dataSize ~136.02 KB/139285, heapSize ~384.41 KB/393640, currentSize=46.96 KB/48090 for 37eda6b0dad56caa0fa6c1ac8208d5b2 in 1276ms, sequenceid=328, compaction requested=false 2024-11-07T12:43:41,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2603): Flush status journal for 37eda6b0dad56caa0fa6c1ac8208d5b2: 2024-11-07T12:43:41,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:41,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=173 2024-11-07T12:43:41,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4169): Remote procedure done, pid=173 2024-11-07T12:43:41,331 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=173, resume processing ppid=172 2024-11-07T12:43:41,331 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=173, ppid=172, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3510 sec 2024-11-07T12:43:41,333 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees in 2.3590 sec 2024-11-07T12:43:41,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:41,415 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 37eda6b0dad56caa0fa6c1ac8208d5b2 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-07T12:43:41,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=A 2024-11-07T12:43:41,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:41,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=B 2024-11-07T12:43:41,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:41,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=C 2024-11-07T12:43:41,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:41,421 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107189390bbc7f5409fba767f607cdd3369_37eda6b0dad56caa0fa6c1ac8208d5b2 is 50, key is test_row_0/A:col10/1730983421413/Put/seqid=0 2024-11-07T12:43:41,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742576_1752 (size=12454) 2024-11-07T12:43:41,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:41,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47962 deadline: 1730983481441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:41,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:41,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:41,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:41,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:41,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47946 deadline: 1730983481445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:41,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:41,452 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:41,452 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:41,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47926 deadline: 1730983481446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:41,452 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:41,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:41,452 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:41,452 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:41,452 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:41,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1730983481447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:41,452 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:41,452 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:41,452 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:41,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:41,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47962 deadline: 1730983481551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:41,556 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:41,556 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:41,556 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:41,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:41,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47946 deadline: 1730983481562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:41,563 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:41,563 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:41,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:41,563 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:41,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1730983481562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:41,563 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:41,563 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:41,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:41,563 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:41,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47926 deadline: 1730983481562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:41,563 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:41,563 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:41,563 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:41,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:41,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47962 deadline: 1730983481761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:41,764 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:41,764 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:41,764 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:41,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:41,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47946 deadline: 1730983481771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:41,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:41,776 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:41,776 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:41,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1730983481772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:41,776 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:41,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:41,776 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:41,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47926 deadline: 1730983481772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:41,776 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:41,776 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:41,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:41,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:41,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:41,827 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:41,831 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107189390bbc7f5409fba767f607cdd3369_37eda6b0dad56caa0fa6c1ac8208d5b2 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107189390bbc7f5409fba767f607cdd3369_37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:41,831 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/52fd63813fa64183b8a214d000fafd90, store: [table=TestAcidGuarantees family=A region=37eda6b0dad56caa0fa6c1ac8208d5b2] 2024-11-07T12:43:41,832 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/52fd63813fa64183b8a214d000fafd90 is 175, key is test_row_0/A:col10/1730983421413/Put/seqid=0 2024-11-07T12:43:41,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742577_1753 (size=31255) 2024-11-07T12:43:41,836 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=342, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/52fd63813fa64183b8a214d000fafd90 2024-11-07T12:43:41,843 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/ffdb01f0cb4f435d9eadf970cbcd0dd1 is 50, key is test_row_0/B:col10/1730983421413/Put/seqid=0 2024-11-07T12:43:41,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742578_1754 (size=12301) 2024-11-07T12:43:42,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:42,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47962 deadline: 1730983482071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:42,076 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:42,076 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:42,076 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:42,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:42,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47946 deadline: 1730983482081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:42,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:42,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1730983482081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:42,085 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:42,085 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:42,085 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:42,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:42,085 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:42,085 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:42,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47926 deadline: 1730983482081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:42,085 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:42,085 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:42,085 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:42,085 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:42,248 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/ffdb01f0cb4f435d9eadf970cbcd0dd1 2024-11-07T12:43:42,255 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/f1fc0d2300c54ca7a6eb8cc8f43b1bc8 is 50, key is test_row_0/C:col10/1730983421413/Put/seqid=0 2024-11-07T12:43:42,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742579_1755 (size=12301) 2024-11-07T12:43:42,269 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/f1fc0d2300c54ca7a6eb8cc8f43b1bc8 2024-11-07T12:43:42,273 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/52fd63813fa64183b8a214d000fafd90 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/52fd63813fa64183b8a214d000fafd90 2024-11-07T12:43:42,276 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/52fd63813fa64183b8a214d000fafd90, entries=150, sequenceid=342, filesize=30.5 K 2024-11-07T12:43:42,277 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/ffdb01f0cb4f435d9eadf970cbcd0dd1 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/ffdb01f0cb4f435d9eadf970cbcd0dd1 2024-11-07T12:43:42,280 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/ffdb01f0cb4f435d9eadf970cbcd0dd1, entries=150, sequenceid=342, filesize=12.0 K 2024-11-07T12:43:42,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/f1fc0d2300c54ca7a6eb8cc8f43b1bc8 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/f1fc0d2300c54ca7a6eb8cc8f43b1bc8 2024-11-07T12:43:42,284 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/f1fc0d2300c54ca7a6eb8cc8f43b1bc8, entries=150, sequenceid=342, filesize=12.0 K 2024-11-07T12:43:42,285 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 37eda6b0dad56caa0fa6c1ac8208d5b2 in 870ms, sequenceid=342, compaction requested=true 2024-11-07T12:43:42,285 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 37eda6b0dad56caa0fa6c1ac8208d5b2: 2024-11-07T12:43:42,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 37eda6b0dad56caa0fa6c1ac8208d5b2:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:43:42,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:42,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 37eda6b0dad56caa0fa6c1ac8208d5b2:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:43:42,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:42,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 37eda6b0dad56caa0fa6c1ac8208d5b2:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:43:42,285 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:43:42,285 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:43:42,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:43:42,286 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:43:42,286 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 37eda6b0dad56caa0fa6c1ac8208d5b2/B is initiating minor compaction (all files) 2024-11-07T12:43:42,286 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 37eda6b0dad56caa0fa6c1ac8208d5b2/B in TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:42,286 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/011fa10c6f0f49c6a487f835a1ba6702, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/8963df44fc5f42adaea6d1f1226e8eef, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/ffdb01f0cb4f435d9eadf970cbcd0dd1] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp, totalSize=36.8 K 2024-11-07T12:43:42,286 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94481 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:43:42,287 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 011fa10c6f0f49c6a487f835a1ba6702, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1730983419011 2024-11-07T12:43:42,287 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): 37eda6b0dad56caa0fa6c1ac8208d5b2/A is initiating minor compaction (all files) 2024-11-07T12:43:42,287 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 37eda6b0dad56caa0fa6c1ac8208d5b2/A in TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:42,287 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/aad8221e321945fe85605d38843ebb2f, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/558d96c6f1314fa085eb857e0bd46a64, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/52fd63813fa64183b8a214d000fafd90] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp, totalSize=92.3 K 2024-11-07T12:43:42,287 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 8963df44fc5f42adaea6d1f1226e8eef, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1730983419117 2024-11-07T12:43:42,287 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:42,287 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. files: [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/aad8221e321945fe85605d38843ebb2f, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/558d96c6f1314fa085eb857e0bd46a64, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/52fd63813fa64183b8a214d000fafd90] 2024-11-07T12:43:42,287 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting ffdb01f0cb4f435d9eadf970cbcd0dd1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1730983421413 2024-11-07T12:43:42,287 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting aad8221e321945fe85605d38843ebb2f, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1730983419011 2024-11-07T12:43:42,288 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 558d96c6f1314fa085eb857e0bd46a64, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1730983419117 2024-11-07T12:43:42,288 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52fd63813fa64183b8a214d000fafd90, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1730983421413 2024-11-07T12:43:42,293 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 37eda6b0dad56caa0fa6c1ac8208d5b2#B#compaction#639 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:43:42,294 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/6a2f1c616a914ad2b09bdf8244091ba8 is 50, key is test_row_0/B:col10/1730983421413/Put/seqid=0 2024-11-07T12:43:42,295 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=37eda6b0dad56caa0fa6c1ac8208d5b2] 2024-11-07T12:43:42,297 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241107694108b811c54dae820f4aa855201a5a_37eda6b0dad56caa0fa6c1ac8208d5b2 store=[table=TestAcidGuarantees family=A region=37eda6b0dad56caa0fa6c1ac8208d5b2] 2024-11-07T12:43:42,299 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241107694108b811c54dae820f4aa855201a5a_37eda6b0dad56caa0fa6c1ac8208d5b2, store=[table=TestAcidGuarantees family=A region=37eda6b0dad56caa0fa6c1ac8208d5b2] 2024-11-07T12:43:42,299 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107694108b811c54dae820f4aa855201a5a_37eda6b0dad56caa0fa6c1ac8208d5b2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=37eda6b0dad56caa0fa6c1ac8208d5b2] 2024-11-07T12:43:42,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742580_1756 (size=13153) 2024-11-07T12:43:42,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742581_1757 (size=4469) 2024-11-07T12:43:42,306 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 37eda6b0dad56caa0fa6c1ac8208d5b2#A#compaction#640 average throughput is 2.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:43:42,306 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/6a2f1c616a914ad2b09bdf8244091ba8 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/6a2f1c616a914ad2b09bdf8244091ba8 2024-11-07T12:43:42,307 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/6bbbf9b699c94bda9f6911c90a4a0a1c is 175, key is test_row_0/A:col10/1730983421413/Put/seqid=0 2024-11-07T12:43:42,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742582_1758 (size=32073) 2024-11-07T12:43:42,311 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 37eda6b0dad56caa0fa6c1ac8208d5b2/B of 37eda6b0dad56caa0fa6c1ac8208d5b2 into 6a2f1c616a914ad2b09bdf8244091ba8(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:43:42,311 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 37eda6b0dad56caa0fa6c1ac8208d5b2: 2024-11-07T12:43:42,311 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., storeName=37eda6b0dad56caa0fa6c1ac8208d5b2/B, priority=13, startTime=1730983422285; duration=0sec 2024-11-07T12:43:42,311 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:43:42,311 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 37eda6b0dad56caa0fa6c1ac8208d5b2:B 2024-11-07T12:43:42,311 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:43:42,313 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:43:42,313 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 37eda6b0dad56caa0fa6c1ac8208d5b2/C is initiating minor compaction (all files) 2024-11-07T12:43:42,313 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 37eda6b0dad56caa0fa6c1ac8208d5b2/C in TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:42,313 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/a1d98a94ed454b478ba1dd30c9c87505, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/e76b8ceadf3d4ab6a5f473d9d40210cf, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/f1fc0d2300c54ca7a6eb8cc8f43b1bc8] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp, totalSize=36.8 K 2024-11-07T12:43:42,313 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting a1d98a94ed454b478ba1dd30c9c87505, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1730983419011 2024-11-07T12:43:42,313 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting e76b8ceadf3d4ab6a5f473d9d40210cf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1730983419117 2024-11-07T12:43:42,314 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting f1fc0d2300c54ca7a6eb8cc8f43b1bc8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1730983421413 2024-11-07T12:43:42,315 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/6bbbf9b699c94bda9f6911c90a4a0a1c as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/6bbbf9b699c94bda9f6911c90a4a0a1c 2024-11-07T12:43:42,320 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 37eda6b0dad56caa0fa6c1ac8208d5b2/A of 37eda6b0dad56caa0fa6c1ac8208d5b2 into 6bbbf9b699c94bda9f6911c90a4a0a1c(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:43:42,320 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 37eda6b0dad56caa0fa6c1ac8208d5b2: 2024-11-07T12:43:42,320 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., storeName=37eda6b0dad56caa0fa6c1ac8208d5b2/A, priority=13, startTime=1730983422285; duration=0sec 2024-11-07T12:43:42,320 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:42,320 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 37eda6b0dad56caa0fa6c1ac8208d5b2:A 2024-11-07T12:43:42,320 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 37eda6b0dad56caa0fa6c1ac8208d5b2#C#compaction#641 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:43:42,321 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/f02c3e6aa67e467993be6b243d2ebeba is 50, key is test_row_0/C:col10/1730983421413/Put/seqid=0 2024-11-07T12:43:42,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742583_1759 (size=13153) 2024-11-07T12:43:42,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:42,583 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 37eda6b0dad56caa0fa6c1ac8208d5b2 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-07T12:43:42,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=A 2024-11-07T12:43:42,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:42,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=B 2024-11-07T12:43:42,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:42,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=C 2024-11-07T12:43:42,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:42,592 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110727011b0f09d2482088da5e31aa82d3b4_37eda6b0dad56caa0fa6c1ac8208d5b2 is 50, key is test_row_0/A:col10/1730983421444/Put/seqid=0 2024-11-07T12:43:42,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742584_1760 (size=14994) 2024-11-07T12:43:42,595 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:42,598 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110727011b0f09d2482088da5e31aa82d3b4_37eda6b0dad56caa0fa6c1ac8208d5b2 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110727011b0f09d2482088da5e31aa82d3b4_37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:42,599 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/b1cbb8f055814c13bebebe2e711525e8, store: [table=TestAcidGuarantees family=A region=37eda6b0dad56caa0fa6c1ac8208d5b2] 2024-11-07T12:43:42,599 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/b1cbb8f055814c13bebebe2e711525e8 is 175, key is test_row_0/A:col10/1730983421444/Put/seqid=0 2024-11-07T12:43:42,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742585_1761 (size=39949) 2024-11-07T12:43:42,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:42,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47962 deadline: 1730983482628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:42,635 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:42,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47946 deadline: 1730983482629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:42,635 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:42,635 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:42,635 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:42,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1730983482630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:42,635 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:42,635 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:42,635 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:42,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:42,635 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:42,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:42,636 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:42,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:42,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47926 deadline: 1730983482631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:42,637 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:42,637 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:42,637 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:42,728 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/f02c3e6aa67e467993be6b243d2ebeba as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/f02c3e6aa67e467993be6b243d2ebeba 2024-11-07T12:43:42,733 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 37eda6b0dad56caa0fa6c1ac8208d5b2/C of 37eda6b0dad56caa0fa6c1ac8208d5b2 into f02c3e6aa67e467993be6b243d2ebeba(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:43:42,733 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 37eda6b0dad56caa0fa6c1ac8208d5b2: 2024-11-07T12:43:42,733 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., storeName=37eda6b0dad56caa0fa6c1ac8208d5b2/C, priority=13, startTime=1730983422285; duration=0sec 2024-11-07T12:43:42,733 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:42,733 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 37eda6b0dad56caa0fa6c1ac8208d5b2:C 2024-11-07T12:43:42,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:42,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47946 deadline: 1730983482742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:42,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:42,746 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:42,746 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:42,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1730983482742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:42,746 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:42,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:42,747 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:42,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47962 deadline: 1730983482742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:42,747 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:42,747 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:42,747 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:42,747 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:42,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:42,747 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:42,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47926 deadline: 1730983482742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:42,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:42,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:42,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:42,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:42,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47946 deadline: 1730983482952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:42,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:42,955 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:42,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47962 deadline: 1730983482952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:42,955 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:42,955 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:42,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:42,955 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:42,955 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:42,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47926 deadline: 1730983482952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:42,955 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:42,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:42,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:42,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:42,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:42,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1730983482952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:42,956 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:42,956 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:42,956 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:43,004 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=364, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/b1cbb8f055814c13bebebe2e711525e8 2024-11-07T12:43:43,011 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/1c44bd143a7d433da3269f3b795bc0d4 is 50, key is test_row_0/B:col10/1730983421444/Put/seqid=0 2024-11-07T12:43:43,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742586_1762 (size=12301) 2024-11-07T12:43:43,015 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/1c44bd143a7d433da3269f3b795bc0d4 2024-11-07T12:43:43,021 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/7f841c78c2214715bd9d2fbd9c828285 is 50, key is test_row_0/C:col10/1730983421444/Put/seqid=0 2024-11-07T12:43:43,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742587_1763 (size=12301) 2024-11-07T12:43:43,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-07T12:43:43,142 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:43:43,143 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:43:43,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] procedure2.ProcedureExecutor(1139): Stored pid=174, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees 2024-11-07T12:43:43,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-07T12:43:43,145 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=174, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:43:43,145 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=174, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:43:43,146 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=175, ppid=174, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:43:43,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-07T12:43:43,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:43,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47946 deadline: 1730983483262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:43,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:43,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:43,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:43,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:43,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47962 deadline: 1730983483262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:43,266 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:43,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:43,266 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:43,266 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:43,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47926 deadline: 1730983483262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:43,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:43,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:43,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:43,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:43,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1730983483262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:43,267 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:43,267 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:43,267 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:43,298 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-07T12:43:43,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:43,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. as already flushing 2024-11-07T12:43:43,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:43,298 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:43,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:43,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:43,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:43,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47984 deadline: 1730983483331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:43,332 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to db9ad1cb6cf9,34293,1730983260217 for 'test_row_1' in 37eda6b0dad56caa0fa6c1ac8208d5b2 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:43:43,333 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:43,333 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:43,333 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:43,425 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/7f841c78c2214715bd9d2fbd9c828285 2024-11-07T12:43:43,429 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/b1cbb8f055814c13bebebe2e711525e8 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/b1cbb8f055814c13bebebe2e711525e8 2024-11-07T12:43:43,432 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/b1cbb8f055814c13bebebe2e711525e8, entries=200, sequenceid=364, filesize=39.0 K 2024-11-07T12:43:43,433 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/1c44bd143a7d433da3269f3b795bc0d4 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/1c44bd143a7d433da3269f3b795bc0d4 2024-11-07T12:43:43,436 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/1c44bd143a7d433da3269f3b795bc0d4, entries=150, sequenceid=364, filesize=12.0 K 2024-11-07T12:43:43,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/7f841c78c2214715bd9d2fbd9c828285 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/7f841c78c2214715bd9d2fbd9c828285 2024-11-07T12:43:43,440 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/7f841c78c2214715bd9d2fbd9c828285, entries=150, sequenceid=364, filesize=12.0 K 2024-11-07T12:43:43,441 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 37eda6b0dad56caa0fa6c1ac8208d5b2 in 858ms, sequenceid=364, compaction requested=false 2024-11-07T12:43:43,441 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 37eda6b0dad56caa0fa6c1ac8208d5b2: 2024-11-07T12:43:43,451 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-07T12:43:43,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:43,451 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2902): Flushing 37eda6b0dad56caa0fa6c1ac8208d5b2 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-07T12:43:43,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=A 2024-11-07T12:43:43,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:43,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=B 2024-11-07T12:43:43,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:43,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=C 2024-11-07T12:43:43,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:43,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411073322b56604ce4f54aaf66c858556ca4a_37eda6b0dad56caa0fa6c1ac8208d5b2 is 50, key is test_row_0/A:col10/1730983422629/Put/seqid=0 2024-11-07T12:43:43,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742588_1764 (size=12454) 2024-11-07T12:43:43,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:43,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-07T12:43:43,466 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411073322b56604ce4f54aaf66c858556ca4a_37eda6b0dad56caa0fa6c1ac8208d5b2 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411073322b56604ce4f54aaf66c858556ca4a_37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:43,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/ea55d8de93be49139efeb4786d041393, store: [table=TestAcidGuarantees family=A region=37eda6b0dad56caa0fa6c1ac8208d5b2] 2024-11-07T12:43:43,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/ea55d8de93be49139efeb4786d041393 is 175, key is test_row_0/A:col10/1730983422629/Put/seqid=0 2024-11-07T12:43:43,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742589_1765 (size=31255) 2024-11-07T12:43:43,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-07T12:43:43,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:43,773 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. as already flushing 2024-11-07T12:43:43,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:43,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47946 deadline: 1730983483844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:43,851 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:43,851 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:43,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:43,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:43,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47926 deadline: 1730983483849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:43,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:43,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1730983483850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:43,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:43,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:43,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:43,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:43,856 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:43,856 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:43,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47962 deadline: 1730983483851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:43,856 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:43,856 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:43,856 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:43,856 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:43,872 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=377, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/ea55d8de93be49139efeb4786d041393 2024-11-07T12:43:43,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/840bafd3df7a4b998f1f45798349f4d0 is 50, key is test_row_0/B:col10/1730983422629/Put/seqid=0 2024-11-07T12:43:43,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742590_1766 (size=12301) 2024-11-07T12:43:43,889 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/840bafd3df7a4b998f1f45798349f4d0 2024-11-07T12:43:43,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/59aca4f5d1644d24b712406f3362734f is 50, key is test_row_0/C:col10/1730983422629/Put/seqid=0 2024-11-07T12:43:43,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742591_1767 (size=12301) 2024-11-07T12:43:43,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:43,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47946 deadline: 1730983483961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:43,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:43,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:43,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:43,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47962 deadline: 1730983483962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:43,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:43,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:43,963 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:43,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47926 deadline: 1730983483962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:43,963 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:43,963 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:43,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:43,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:43,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1730983483962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:43,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:43,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:43,963 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:43,963 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:43,963 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:44,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:44,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47946 deadline: 1730983484171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:44,175 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:44,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:44,175 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:44,175 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:44,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47962 deadline: 1730983484172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:44,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:44,176 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:44,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47926 deadline: 1730983484172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:44,176 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:44,176 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:44,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:44,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:44,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:44,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1730983484172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:44,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:44,176 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:44,176 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:44,176 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:44,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-07T12:43:44,299 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/59aca4f5d1644d24b712406f3362734f 2024-11-07T12:43:44,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/ea55d8de93be49139efeb4786d041393 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/ea55d8de93be49139efeb4786d041393 2024-11-07T12:43:44,306 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/ea55d8de93be49139efeb4786d041393, entries=150, sequenceid=377, filesize=30.5 K 2024-11-07T12:43:44,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/840bafd3df7a4b998f1f45798349f4d0 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/840bafd3df7a4b998f1f45798349f4d0 2024-11-07T12:43:44,310 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/840bafd3df7a4b998f1f45798349f4d0, entries=150, sequenceid=377, filesize=12.0 K 2024-11-07T12:43:44,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/59aca4f5d1644d24b712406f3362734f as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/59aca4f5d1644d24b712406f3362734f 2024-11-07T12:43:44,313 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/59aca4f5d1644d24b712406f3362734f, entries=150, sequenceid=377, filesize=12.0 K 2024-11-07T12:43:44,314 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 37eda6b0dad56caa0fa6c1ac8208d5b2 in 863ms, sequenceid=377, compaction requested=true 2024-11-07T12:43:44,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2603): Flush status journal for 37eda6b0dad56caa0fa6c1ac8208d5b2: 2024-11-07T12:43:44,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:44,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=175 2024-11-07T12:43:44,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4169): Remote procedure done, pid=175 2024-11-07T12:43:44,317 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=175, resume processing ppid=174 2024-11-07T12:43:44,317 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=175, ppid=174, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1710 sec 2024-11-07T12:43:44,319 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=174, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees in 1.1750 sec 2024-11-07T12:43:44,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:44,485 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 37eda6b0dad56caa0fa6c1ac8208d5b2 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-07T12:43:44,486 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1730983423842, max timestamp=1730983424483Num uniques 859; 2024-11-07T12:43:44,486 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/db9ad1cb6cf9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-07T12:43:44,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=A 2024-11-07T12:43:44,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:43:44,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=B 2024-11-07T12:43:44,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:43:44,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=C 2024-11-07T12:43:44,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-07T12:43:44,493 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411071b9bad08d1cb4e5189e9f672d7927302_37eda6b0dad56caa0fa6c1ac8208d5b2 is 50, key is test_row_0/A:col10/1730983424484/Put/seqid=0 2024-11-07T12:43:44,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742592_1768 (size=17534) 2024-11-07T12:43:44,498 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:44,503 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411071b9bad08d1cb4e5189e9f672d7927302_37eda6b0dad56caa0fa6c1ac8208d5b2 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411071b9bad08d1cb4e5189e9f672d7927302_37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:44,504 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/4cc721003c4b4555a10eaf04b6254b5b, store: [table=TestAcidGuarantees family=A region=37eda6b0dad56caa0fa6c1ac8208d5b2] 2024-11-07T12:43:44,504 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/4cc721003c4b4555a10eaf04b6254b5b is 175, key is test_row_0/A:col10/1730983424484/Put/seqid=0 2024-11-07T12:43:44,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742593_1769 (size=48639) 2024-11-07T12:43:44,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:44,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47946 deadline: 1730983484506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:44,513 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:44,513 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:44,513 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:44,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:44,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47926 deadline: 1730983484511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:44,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:44,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:44,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:44,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47962 deadline: 1730983484513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:44,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:44,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:44,520 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:44,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1730983484513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:44,521 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:44,521 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:44,521 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:44,521 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:44,521 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:44,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:44,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47946 deadline: 1730983484622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:44,626 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:44,626 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:44,626 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:44,635 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:44,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47926 deadline: 1730983484631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:44,635 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:44,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47962 deadline: 1730983484631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:44,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:44,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:44,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:44,635 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:44,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1730983484631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:44,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:44,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:44,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:44,635 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:44,636 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:44,636 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:44,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:44,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47946 deadline: 1730983484831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:44,836 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:44,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:44,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:44,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:44,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47926 deadline: 1730983484841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:44,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:44,846 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:44,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47962 deadline: 1730983484842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:44,846 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:44,846 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:44,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:44,846 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:44,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1730983484842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:44,846 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:44,846 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:44,846 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:44,847 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:44,847 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:44,908 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=401, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/4cc721003c4b4555a10eaf04b6254b5b 2024-11-07T12:43:44,915 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/dd4f586e079c4913b8d78fa2476451eb is 50, key is test_row_0/B:col10/1730983424484/Put/seqid=0 2024-11-07T12:43:44,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742594_1770 (size=12301) 2024-11-07T12:43:45,146 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:45,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47946 deadline: 1730983485142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:45,146 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:45,146 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:45,146 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:45,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:45,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47926 deadline: 1730983485151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:45,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:45,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:45,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:45,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:45,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47962 deadline: 1730983485151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:45,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:45,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1730983485151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:45,155 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:45,155 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:45,155 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:45,155 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:45,156 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:45,156 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:45,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-07T12:43:45,292 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:43:45,292 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:43:45,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] procedure2.ProcedureExecutor(1139): Stored pid=176, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=176, table=TestAcidGuarantees 2024-11-07T12:43:45,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-07T12:43:45,294 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=176, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=176, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:43:45,294 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=176, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=176, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:43:45,295 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=177, ppid=176, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:43:45,318 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/dd4f586e079c4913b8d78fa2476451eb 2024-11-07T12:43:45,331 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/64bc26786f3a4f04b4aca537a947b438 is 50, key is test_row_0/C:col10/1730983424484/Put/seqid=0 2024-11-07T12:43:45,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742595_1771 (size=12301) 2024-11-07T12:43:45,337 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/64bc26786f3a4f04b4aca537a947b438 2024-11-07T12:43:45,343 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/4cc721003c4b4555a10eaf04b6254b5b as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/4cc721003c4b4555a10eaf04b6254b5b 2024-11-07T12:43:45,351 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/4cc721003c4b4555a10eaf04b6254b5b, entries=250, sequenceid=401, filesize=47.5 K 2024-11-07T12:43:45,352 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/dd4f586e079c4913b8d78fa2476451eb as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/dd4f586e079c4913b8d78fa2476451eb 2024-11-07T12:43:45,355 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/dd4f586e079c4913b8d78fa2476451eb, entries=150, sequenceid=401, filesize=12.0 K 2024-11-07T12:43:45,356 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/64bc26786f3a4f04b4aca537a947b438 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/64bc26786f3a4f04b4aca537a947b438 2024-11-07T12:43:45,360 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/64bc26786f3a4f04b4aca537a947b438, entries=150, sequenceid=401, filesize=12.0 K 2024-11-07T12:43:45,361 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~109.58 KB/112210, heapSize ~321.83 KB/329552, currentSize=73.80 KB/75570 for 37eda6b0dad56caa0fa6c1ac8208d5b2 in 876ms, sequenceid=401, compaction requested=true 2024-11-07T12:43:45,361 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 37eda6b0dad56caa0fa6c1ac8208d5b2: 2024-11-07T12:43:45,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 37eda6b0dad56caa0fa6c1ac8208d5b2:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:43:45,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:45,361 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:43:45,361 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:43:45,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 37eda6b0dad56caa0fa6c1ac8208d5b2:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:43:45,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:45,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 37eda6b0dad56caa0fa6c1ac8208d5b2:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:43:45,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:43:45,363 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50056 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:43:45,363 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 37eda6b0dad56caa0fa6c1ac8208d5b2/B is initiating minor compaction (all files) 2024-11-07T12:43:45,363 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 37eda6b0dad56caa0fa6c1ac8208d5b2/B in TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:45,363 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/6a2f1c616a914ad2b09bdf8244091ba8, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/1c44bd143a7d433da3269f3b795bc0d4, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/840bafd3df7a4b998f1f45798349f4d0, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/dd4f586e079c4913b8d78fa2476451eb] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp, totalSize=48.9 K 2024-11-07T12:43:45,363 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 151916 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:43:45,363 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): 37eda6b0dad56caa0fa6c1ac8208d5b2/A is initiating minor compaction (all files) 2024-11-07T12:43:45,364 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 37eda6b0dad56caa0fa6c1ac8208d5b2/A in TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:45,364 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/6bbbf9b699c94bda9f6911c90a4a0a1c, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/b1cbb8f055814c13bebebe2e711525e8, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/ea55d8de93be49139efeb4786d041393, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/4cc721003c4b4555a10eaf04b6254b5b] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp, totalSize=148.4 K 2024-11-07T12:43:45,364 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:45,364 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. files: [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/6bbbf9b699c94bda9f6911c90a4a0a1c, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/b1cbb8f055814c13bebebe2e711525e8, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/ea55d8de93be49139efeb4786d041393, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/4cc721003c4b4555a10eaf04b6254b5b] 2024-11-07T12:43:45,364 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a2f1c616a914ad2b09bdf8244091ba8, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1730983421413 2024-11-07T12:43:45,365 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6bbbf9b699c94bda9f6911c90a4a0a1c, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1730983421413 2024-11-07T12:43:45,366 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c44bd143a7d433da3269f3b795bc0d4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1730983421444 2024-11-07T12:43:45,367 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1cbb8f055814c13bebebe2e711525e8, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1730983421435 2024-11-07T12:43:45,367 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 840bafd3df7a4b998f1f45798349f4d0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1730983422594 2024-11-07T12:43:45,367 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea55d8de93be49139efeb4786d041393, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1730983422594 2024-11-07T12:43:45,367 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting dd4f586e079c4913b8d78fa2476451eb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1730983423842 2024-11-07T12:43:45,368 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4cc721003c4b4555a10eaf04b6254b5b, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1730983423842 2024-11-07T12:43:45,381 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 37eda6b0dad56caa0fa6c1ac8208d5b2#B#compaction#651 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:43:45,381 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/71ddc6e133934c648245167de8d1feda is 50, key is test_row_0/B:col10/1730983424484/Put/seqid=0 2024-11-07T12:43:45,390 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=37eda6b0dad56caa0fa6c1ac8208d5b2] 2024-11-07T12:43:45,397 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411077e4abfe652034d55b74d93a614219842_37eda6b0dad56caa0fa6c1ac8208d5b2 store=[table=TestAcidGuarantees family=A region=37eda6b0dad56caa0fa6c1ac8208d5b2] 2024-11-07T12:43:45,400 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411077e4abfe652034d55b74d93a614219842_37eda6b0dad56caa0fa6c1ac8208d5b2, store=[table=TestAcidGuarantees family=A region=37eda6b0dad56caa0fa6c1ac8208d5b2] 2024-11-07T12:43:45,400 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411077e4abfe652034d55b74d93a614219842_37eda6b0dad56caa0fa6c1ac8208d5b2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=37eda6b0dad56caa0fa6c1ac8208d5b2] 2024-11-07T12:43:45,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-07T12:43:45,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742596_1772 (size=13289) 2024-11-07T12:43:45,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742597_1773 (size=4469) 2024-11-07T12:43:45,411 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 37eda6b0dad56caa0fa6c1ac8208d5b2#A#compaction#652 average throughput is 1.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:43:45,411 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/b2824d53988b417e8571cc9b390a7e29 is 175, key is test_row_0/A:col10/1730983424484/Put/seqid=0 2024-11-07T12:43:45,415 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/71ddc6e133934c648245167de8d1feda as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/71ddc6e133934c648245167de8d1feda 2024-11-07T12:43:45,421 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 37eda6b0dad56caa0fa6c1ac8208d5b2/B of 37eda6b0dad56caa0fa6c1ac8208d5b2 into 71ddc6e133934c648245167de8d1feda(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:43:45,421 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 37eda6b0dad56caa0fa6c1ac8208d5b2: 2024-11-07T12:43:45,421 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., storeName=37eda6b0dad56caa0fa6c1ac8208d5b2/B, priority=12, startTime=1730983425361; duration=0sec 2024-11-07T12:43:45,421 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:43:45,421 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 37eda6b0dad56caa0fa6c1ac8208d5b2:B 2024-11-07T12:43:45,421 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:43:45,424 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50056 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:43:45,424 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 37eda6b0dad56caa0fa6c1ac8208d5b2/C is initiating minor compaction (all files) 2024-11-07T12:43:45,424 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 37eda6b0dad56caa0fa6c1ac8208d5b2/C in TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:45,424 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/f02c3e6aa67e467993be6b243d2ebeba, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/7f841c78c2214715bd9d2fbd9c828285, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/59aca4f5d1644d24b712406f3362734f, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/64bc26786f3a4f04b4aca537a947b438] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp, totalSize=48.9 K 2024-11-07T12:43:45,425 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting f02c3e6aa67e467993be6b243d2ebeba, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1730983421413 2024-11-07T12:43:45,425 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f841c78c2214715bd9d2fbd9c828285, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1730983421444 2024-11-07T12:43:45,425 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 59aca4f5d1644d24b712406f3362734f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1730983422594 2024-11-07T12:43:45,426 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 64bc26786f3a4f04b4aca537a947b438, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1730983423842 2024-11-07T12:43:45,447 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=177 2024-11-07T12:43:45,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:45,448 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegion(2902): Flushing 37eda6b0dad56caa0fa6c1ac8208d5b2 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-07T12:43:45,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=A 2024-11-07T12:43:45,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:45,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=B 2024-11-07T12:43:45,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:45,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=C 2024-11-07T12:43:45,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:45,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742598_1774 (size=32209) 2024-11-07T12:43:45,451 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 37eda6b0dad56caa0fa6c1ac8208d5b2#C#compaction#653 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:43:45,452 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/87654538469d4acd8cc405dc88911f24 is 50, key is test_row_0/C:col10/1730983424484/Put/seqid=0 2024-11-07T12:43:45,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110737a83c12357e4e978b6278766ab49030_37eda6b0dad56caa0fa6c1ac8208d5b2 is 50, key is test_row_0/A:col10/1730983424501/Put/seqid=0 2024-11-07T12:43:45,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742599_1775 (size=13289) 2024-11-07T12:43:45,499 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/87654538469d4acd8cc405dc88911f24 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/87654538469d4acd8cc405dc88911f24 2024-11-07T12:43:45,505 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 37eda6b0dad56caa0fa6c1ac8208d5b2/C of 37eda6b0dad56caa0fa6c1ac8208d5b2 into 87654538469d4acd8cc405dc88911f24(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:43:45,505 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 37eda6b0dad56caa0fa6c1ac8208d5b2: 2024-11-07T12:43:45,505 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., storeName=37eda6b0dad56caa0fa6c1ac8208d5b2/C, priority=12, startTime=1730983425362; duration=0sec 2024-11-07T12:43:45,505 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:45,505 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 37eda6b0dad56caa0fa6c1ac8208d5b2:C 2024-11-07T12:43:45,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742600_1776 (size=12454) 2024-11-07T12:43:45,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:45,515 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110737a83c12357e4e978b6278766ab49030_37eda6b0dad56caa0fa6c1ac8208d5b2 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110737a83c12357e4e978b6278766ab49030_37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:45,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/894f216a2a9a41439070984dcab56a8d, store: [table=TestAcidGuarantees family=A region=37eda6b0dad56caa0fa6c1ac8208d5b2] 2024-11-07T12:43:45,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/894f216a2a9a41439070984dcab56a8d is 175, key is test_row_0/A:col10/1730983424501/Put/seqid=0 2024-11-07T12:43:45,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742601_1777 (size=31255) 2024-11-07T12:43:45,541 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=416, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/894f216a2a9a41439070984dcab56a8d 2024-11-07T12:43:45,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/78f6b616cb314a1aa243f125e9498a9c is 50, key is test_row_0/B:col10/1730983424501/Put/seqid=0 2024-11-07T12:43:45,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742602_1778 (size=12301) 2024-11-07T12:43:45,574 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/78f6b616cb314a1aa243f125e9498a9c 2024-11-07T12:43:45,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/d2eab486a149417aaefcd5bf7582c9f1 is 50, key is test_row_0/C:col10/1730983424501/Put/seqid=0 2024-11-07T12:43:45,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742603_1779 (size=12301) 2024-11-07T12:43:45,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-07T12:43:45,654 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. as already flushing 2024-11-07T12:43:45,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:45,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:45,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47926 deadline: 1730983485679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:45,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:45,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:45,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:45,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:45,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47962 deadline: 1730983485681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:45,687 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:45,687 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:45,687 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:45,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:45,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1730983485682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:45,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:45,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47946 deadline: 1730983485683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:45,692 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:45,692 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:45,692 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:45,693 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:45,693 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:45,693 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:45,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:45,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47962 deadline: 1730983485792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:45,796 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:45,796 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:45,796 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:45,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47926 deadline: 1730983485792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:45,796 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:45,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:45,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:45,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:45,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:45,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47946 deadline: 1730983485801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:45,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:45,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:45,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:45,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:45,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1730983485801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:45,804 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:45,804 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:45,804 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:45,858 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/b2824d53988b417e8571cc9b390a7e29 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/b2824d53988b417e8571cc9b390a7e29 2024-11-07T12:43:45,866 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 37eda6b0dad56caa0fa6c1ac8208d5b2/A of 37eda6b0dad56caa0fa6c1ac8208d5b2 into b2824d53988b417e8571cc9b390a7e29(size=31.5 K), total size for store is 31.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:43:45,866 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 37eda6b0dad56caa0fa6c1ac8208d5b2: 2024-11-07T12:43:45,866 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., storeName=37eda6b0dad56caa0fa6c1ac8208d5b2/A, priority=12, startTime=1730983425361; duration=0sec 2024-11-07T12:43:45,866 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:45,866 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 37eda6b0dad56caa0fa6c1ac8208d5b2:A 2024-11-07T12:43:45,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-07T12:43:45,991 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/d2eab486a149417aaefcd5bf7582c9f1 2024-11-07T12:43:45,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/894f216a2a9a41439070984dcab56a8d as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/894f216a2a9a41439070984dcab56a8d 2024-11-07T12:43:45,999 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/894f216a2a9a41439070984dcab56a8d, entries=150, sequenceid=416, filesize=30.5 K 2024-11-07T12:43:45,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/78f6b616cb314a1aa243f125e9498a9c as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/78f6b616cb314a1aa243f125e9498a9c 2024-11-07T12:43:46,004 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/78f6b616cb314a1aa243f125e9498a9c, entries=150, sequenceid=416, filesize=12.0 K 2024-11-07T12:43:46,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/d2eab486a149417aaefcd5bf7582c9f1 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/d2eab486a149417aaefcd5bf7582c9f1 2024-11-07T12:43:46,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:46,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47962 deadline: 1730983486002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:46,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:46,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:46,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:46,009 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/d2eab486a149417aaefcd5bf7582c9f1, entries=150, sequenceid=416, filesize=12.0 K 2024-11-07T12:43:46,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:46,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47926 deadline: 1730983486003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:46,010 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:46,010 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:46,010 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:46,010 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 37eda6b0dad56caa0fa6c1ac8208d5b2 in 562ms, sequenceid=416, compaction requested=false 2024-11-07T12:43:46,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegion(2603): Flush status journal for 37eda6b0dad56caa0fa6c1ac8208d5b2: 2024-11-07T12:43:46,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:46,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=177 2024-11-07T12:43:46,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4169): Remote procedure done, pid=177 2024-11-07T12:43:46,014 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=177, resume processing ppid=176 2024-11-07T12:43:46,014 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=177, ppid=176, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 716 msec 2024-11-07T12:43:46,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:46,015 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 37eda6b0dad56caa0fa6c1ac8208d5b2 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-07T12:43:46,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=A 2024-11-07T12:43:46,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:46,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=B 2024-11-07T12:43:46,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:46,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=C 2024-11-07T12:43:46,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:46,017 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=176, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=176, table=TestAcidGuarantees in 723 msec 2024-11-07T12:43:46,022 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107e909ba759e864dcda1c3d9625bab982c_37eda6b0dad56caa0fa6c1ac8208d5b2 is 50, key is test_row_0/A:col10/1730983426014/Put/seqid=0 2024-11-07T12:43:46,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742604_1780 (size=14994) 2024-11-07T12:43:46,038 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:46,041 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107e909ba759e864dcda1c3d9625bab982c_37eda6b0dad56caa0fa6c1ac8208d5b2 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107e909ba759e864dcda1c3d9625bab982c_37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:46,042 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/5cb7baf496164698b2cd7e18d18acf12, store: [table=TestAcidGuarantees family=A region=37eda6b0dad56caa0fa6c1ac8208d5b2] 2024-11-07T12:43:46,043 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/5cb7baf496164698b2cd7e18d18acf12 is 175, key is test_row_0/A:col10/1730983426014/Put/seqid=0 2024-11-07T12:43:46,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:46,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47946 deadline: 1730983486039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:46,045 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:46,045 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:46,045 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:46,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:46,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1730983486043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:46,050 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:46,050 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:46,050 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:46,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742605_1781 (size=39949) 2024-11-07T12:43:46,062 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=438, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/5cb7baf496164698b2cd7e18d18acf12 2024-11-07T12:43:46,068 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/183f04cfa6d8449e93fd160dbc88ac13 is 50, key is test_row_0/B:col10/1730983426014/Put/seqid=0 2024-11-07T12:43:46,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742606_1782 (size=12301) 2024-11-07T12:43:46,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:46,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47946 deadline: 1730983486154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:46,157 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:46,157 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:46,157 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:46,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:46,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1730983486155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:46,158 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:46,158 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:46,158 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:46,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:46,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47962 deadline: 1730983486312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:46,316 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:46,317 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:46,317 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:46,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:46,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47926 deadline: 1730983486321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:46,326 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:46,326 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:46,326 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:46,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:46,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47946 deadline: 1730983486361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:46,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:46,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1730983486361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:46,366 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:46,366 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:46,366 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:46,366 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:46,366 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:46,366 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:46,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-07T12:43:46,443 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:43:46,443 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:43:46,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] procedure2.ProcedureExecutor(1139): Stored pid=178, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=178, table=TestAcidGuarantees 2024-11-07T12:43:46,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-11-07T12:43:46,445 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=178, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=178, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:43:46,446 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=178, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=178, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:43:46,446 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=179, ppid=178, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:43:46,473 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=438 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/183f04cfa6d8449e93fd160dbc88ac13 2024-11-07T12:43:46,491 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/e0220e56e81c4e47a34527d9dd8281b8 is 50, key is test_row_0/C:col10/1730983426014/Put/seqid=0 2024-11-07T12:43:46,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742607_1783 (size=12301) 2024-11-07T12:43:46,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-11-07T12:43:46,600 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=179 2024-11-07T12:43:46,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:46,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. as already flushing 2024-11-07T12:43:46,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:46,601 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] handler.RSProcedureHandler(58): pid=179 java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:46,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=179 java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:46,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=179 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:46,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:46,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1730983486672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:46,676 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:46,676 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:46,676 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:46,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:46,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47946 deadline: 1730983486672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:46,677 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:46,677 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:46,677 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:46,755 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=179 2024-11-07T12:43:46,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:46,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. as already flushing 2024-11-07T12:43:46,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:46,756 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] handler.RSProcedureHandler(58): pid=179 java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:46,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=179 java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:46,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=179 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:46,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-11-07T12:43:46,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:46,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47962 deadline: 1730983486822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:46,823 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:46,823 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:46,823 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:46,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:46,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47926 deadline: 1730983486832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:46,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:46,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:46,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:46,909 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=179 2024-11-07T12:43:46,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:46,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. as already flushing 2024-11-07T12:43:46,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:46,909 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] handler.RSProcedureHandler(58): pid=179 java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:46,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=179 java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:46,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=179 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:46,929 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=438 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/e0220e56e81c4e47a34527d9dd8281b8 2024-11-07T12:43:46,943 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/5cb7baf496164698b2cd7e18d18acf12 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/5cb7baf496164698b2cd7e18d18acf12 2024-11-07T12:43:46,947 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/5cb7baf496164698b2cd7e18d18acf12, entries=200, sequenceid=438, filesize=39.0 K 2024-11-07T12:43:46,948 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/183f04cfa6d8449e93fd160dbc88ac13 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/183f04cfa6d8449e93fd160dbc88ac13 2024-11-07T12:43:46,952 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/183f04cfa6d8449e93fd160dbc88ac13, entries=150, sequenceid=438, filesize=12.0 K 2024-11-07T12:43:46,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/e0220e56e81c4e47a34527d9dd8281b8 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/e0220e56e81c4e47a34527d9dd8281b8 2024-11-07T12:43:46,958 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/e0220e56e81c4e47a34527d9dd8281b8, entries=150, sequenceid=438, filesize=12.0 K 2024-11-07T12:43:46,959 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 37eda6b0dad56caa0fa6c1ac8208d5b2 in 944ms, sequenceid=438, compaction requested=true 2024-11-07T12:43:46,959 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 37eda6b0dad56caa0fa6c1ac8208d5b2: 2024-11-07T12:43:46,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 37eda6b0dad56caa0fa6c1ac8208d5b2:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:43:46,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:46,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 37eda6b0dad56caa0fa6c1ac8208d5b2:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:43:46,959 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:43:46,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:43:46,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 37eda6b0dad56caa0fa6c1ac8208d5b2:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:43:46,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-07T12:43:46,959 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:43:46,960 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103413 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:43:46,960 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:43:46,960 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 37eda6b0dad56caa0fa6c1ac8208d5b2/B is initiating minor compaction (all files) 2024-11-07T12:43:46,960 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): 37eda6b0dad56caa0fa6c1ac8208d5b2/A is initiating minor compaction (all files) 2024-11-07T12:43:46,960 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 37eda6b0dad56caa0fa6c1ac8208d5b2/B in TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:46,960 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 37eda6b0dad56caa0fa6c1ac8208d5b2/A in TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:46,960 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/71ddc6e133934c648245167de8d1feda, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/78f6b616cb314a1aa243f125e9498a9c, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/183f04cfa6d8449e93fd160dbc88ac13] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp, totalSize=37.0 K 2024-11-07T12:43:46,960 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/b2824d53988b417e8571cc9b390a7e29, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/894f216a2a9a41439070984dcab56a8d, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/5cb7baf496164698b2cd7e18d18acf12] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp, totalSize=101.0 K 2024-11-07T12:43:46,961 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:46,961 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. files: [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/b2824d53988b417e8571cc9b390a7e29, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/894f216a2a9a41439070984dcab56a8d, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/5cb7baf496164698b2cd7e18d18acf12] 2024-11-07T12:43:46,961 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 71ddc6e133934c648245167de8d1feda, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1730983423842 2024-11-07T12:43:46,961 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2824d53988b417e8571cc9b390a7e29, keycount=150, bloomtype=ROW, size=31.5 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1730983423842 2024-11-07T12:43:46,962 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 78f6b616cb314a1aa243f125e9498a9c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1730983424501 2024-11-07T12:43:46,962 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 183f04cfa6d8449e93fd160dbc88ac13, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1730983425679 2024-11-07T12:43:46,962 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 894f216a2a9a41439070984dcab56a8d, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1730983424501 2024-11-07T12:43:46,963 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5cb7baf496164698b2cd7e18d18acf12, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1730983425679 2024-11-07T12:43:46,981 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=37eda6b0dad56caa0fa6c1ac8208d5b2] 2024-11-07T12:43:46,999 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 37eda6b0dad56caa0fa6c1ac8208d5b2#B#compaction#661 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:43:46,999 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/0761afa32d974e6d984049e1e5e617a8 is 50, key is test_row_0/B:col10/1730983426014/Put/seqid=0 2024-11-07T12:43:47,008 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411072dffbb3dcac845ccbc898cb8b37df2de_37eda6b0dad56caa0fa6c1ac8208d5b2 store=[table=TestAcidGuarantees family=A region=37eda6b0dad56caa0fa6c1ac8208d5b2] 2024-11-07T12:43:47,011 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411072dffbb3dcac845ccbc898cb8b37df2de_37eda6b0dad56caa0fa6c1ac8208d5b2, store=[table=TestAcidGuarantees family=A region=37eda6b0dad56caa0fa6c1ac8208d5b2] 2024-11-07T12:43:47,011 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411072dffbb3dcac845ccbc898cb8b37df2de_37eda6b0dad56caa0fa6c1ac8208d5b2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=37eda6b0dad56caa0fa6c1ac8208d5b2] 2024-11-07T12:43:47,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742608_1784 (size=13391) 2024-11-07T12:43:47,063 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=179 2024-11-07T12:43:47,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:47,063 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegion(2902): Flushing 37eda6b0dad56caa0fa6c1ac8208d5b2 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-07T12:43:47,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=A 2024-11-07T12:43:47,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:47,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=B 2024-11-07T12:43:47,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:47,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=C 2024-11-07T12:43:47,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:47,068 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/0761afa32d974e6d984049e1e5e617a8 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/0761afa32d974e6d984049e1e5e617a8 2024-11-07T12:43:47,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-11-07T12:43:47,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742609_1785 (size=4469) 2024-11-07T12:43:47,080 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 37eda6b0dad56caa0fa6c1ac8208d5b2/B of 37eda6b0dad56caa0fa6c1ac8208d5b2 into 0761afa32d974e6d984049e1e5e617a8(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:43:47,080 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 37eda6b0dad56caa0fa6c1ac8208d5b2: 2024-11-07T12:43:47,080 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., storeName=37eda6b0dad56caa0fa6c1ac8208d5b2/B, priority=13, startTime=1730983426959; duration=0sec 2024-11-07T12:43:47,080 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:43:47,080 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 37eda6b0dad56caa0fa6c1ac8208d5b2:B 2024-11-07T12:43:47,080 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:43:47,081 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:43:47,081 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 37eda6b0dad56caa0fa6c1ac8208d5b2/C is initiating minor compaction (all files) 2024-11-07T12:43:47,081 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 37eda6b0dad56caa0fa6c1ac8208d5b2/C in TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:47,082 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/87654538469d4acd8cc405dc88911f24, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/d2eab486a149417aaefcd5bf7582c9f1, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/e0220e56e81c4e47a34527d9dd8281b8] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp, totalSize=37.0 K 2024-11-07T12:43:47,082 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 87654538469d4acd8cc405dc88911f24, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1730983423842 2024-11-07T12:43:47,083 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting d2eab486a149417aaefcd5bf7582c9f1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1730983424501 2024-11-07T12:43:47,083 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting e0220e56e81c4e47a34527d9dd8281b8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1730983425679 2024-11-07T12:43:47,103 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 37eda6b0dad56caa0fa6c1ac8208d5b2#C#compaction#662 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:43:47,103 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/0fc56b1d51364298a349e0d2758ab3c3 is 50, key is test_row_0/C:col10/1730983426014/Put/seqid=0 2024-11-07T12:43:47,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107bf62a513b19e4f6dace68e1e4312183d_37eda6b0dad56caa0fa6c1ac8208d5b2 is 50, key is test_row_0/A:col10/1730983426041/Put/seqid=0 2024-11-07T12:43:47,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742610_1786 (size=13391) 2024-11-07T12:43:47,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742611_1787 (size=12454) 2024-11-07T12:43:47,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:47,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. as already flushing 2024-11-07T12:43:47,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:47,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47946 deadline: 1730983487225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:47,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:47,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:47,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:47,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:47,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1730983487228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:47,233 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:47,233 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:47,233 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:47,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:47,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47946 deadline: 1730983487342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:47,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:47,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1730983487342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:47,345 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:47,345 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:47,345 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:47,346 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:47,346 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:47,346 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:47,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:47,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47984 deadline: 1730983487372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:47,376 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to db9ad1cb6cf9,34293,1730983260217 for 'test_row_1' in 37eda6b0dad56caa0fa6c1ac8208d5b2 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8249 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:43:47,376 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:47,376 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:47,376 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:47,477 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 37eda6b0dad56caa0fa6c1ac8208d5b2#A#compaction#660 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:43:47,477 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/604166de2dcd4e6fa3831797fdcae72b is 175, key is test_row_0/A:col10/1730983426014/Put/seqid=0 2024-11-07T12:43:47,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742612_1788 (size=32311) 2024-11-07T12:43:47,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:47,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1730983487552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:47,556 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:47,556 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:47,556 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:47,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:47,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47946 deadline: 1730983487552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:47,556 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:47,557 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:47,557 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:47,570 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/0fc56b1d51364298a349e0d2758ab3c3 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/0fc56b1d51364298a349e0d2758ab3c3 2024-11-07T12:43:47,575 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 37eda6b0dad56caa0fa6c1ac8208d5b2/C of 37eda6b0dad56caa0fa6c1ac8208d5b2 into 0fc56b1d51364298a349e0d2758ab3c3(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:43:47,576 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 37eda6b0dad56caa0fa6c1ac8208d5b2: 2024-11-07T12:43:47,576 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., storeName=37eda6b0dad56caa0fa6c1ac8208d5b2/C, priority=13, startTime=1730983426959; duration=0sec 2024-11-07T12:43:47,576 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:47,576 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 37eda6b0dad56caa0fa6c1ac8208d5b2:C 2024-11-07T12:43:47,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:47,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-11-07T12:43:47,583 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107bf62a513b19e4f6dace68e1e4312183d_37eda6b0dad56caa0fa6c1ac8208d5b2 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107bf62a513b19e4f6dace68e1e4312183d_37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:47,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/9de6a23c0b334b6db88a00c9f7ac5aec, store: [table=TestAcidGuarantees family=A region=37eda6b0dad56caa0fa6c1ac8208d5b2] 2024-11-07T12:43:47,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/9de6a23c0b334b6db88a00c9f7ac5aec is 175, key is test_row_0/A:col10/1730983426041/Put/seqid=0 2024-11-07T12:43:47,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742613_1789 (size=31255) 2024-11-07T12:43:47,598 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=450, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/9de6a23c0b334b6db88a00c9f7ac5aec 2024-11-07T12:43:47,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/20b308226f694491aea04304eb607ef5 is 50, key is test_row_0/B:col10/1730983426041/Put/seqid=0 2024-11-07T12:43:47,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742614_1790 (size=12301) 2024-11-07T12:43:47,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:47,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47962 deadline: 1730983487842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:47,844 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to db9ad1cb6cf9,34293,1730983260217 for 'test_row_2' in 37eda6b0dad56caa0fa6c1ac8208d5b2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:43:47,845 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:47,845 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:47,845 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:47,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:47,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47926 deadline: 1730983487852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:47,856 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to db9ad1cb6cf9,34293,1730983260217 for 'test_row_0' in 37eda6b0dad56caa0fa6c1ac8208d5b2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:43:47,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:47,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:47,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:47,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:47,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1730983487861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:47,865 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:47,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:47,865 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:47,865 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:47,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47946 deadline: 1730983487861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:47,873 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:47,873 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:47,873 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:47,887 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/604166de2dcd4e6fa3831797fdcae72b as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/604166de2dcd4e6fa3831797fdcae72b 2024-11-07T12:43:47,891 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 37eda6b0dad56caa0fa6c1ac8208d5b2/A of 37eda6b0dad56caa0fa6c1ac8208d5b2 into 604166de2dcd4e6fa3831797fdcae72b(size=31.6 K), total size for store is 31.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:43:47,891 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 37eda6b0dad56caa0fa6c1ac8208d5b2: 2024-11-07T12:43:47,892 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., storeName=37eda6b0dad56caa0fa6c1ac8208d5b2/A, priority=13, startTime=1730983426959; duration=0sec 2024-11-07T12:43:47,892 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:47,892 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 37eda6b0dad56caa0fa6c1ac8208d5b2:A 2024-11-07T12:43:48,019 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/20b308226f694491aea04304eb607ef5 2024-11-07T12:43:48,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/0a7639e03b3b43c78962d198da874092 is 50, key is test_row_0/C:col10/1730983426041/Put/seqid=0 2024-11-07T12:43:48,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742615_1791 (size=12301) 2024-11-07T12:43:48,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:48,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1730983488371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:48,374 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:48,374 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:48,374 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:48,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:48,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47946 deadline: 1730983488381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:48,384 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:48,384 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:48,384 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:48,429 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/0a7639e03b3b43c78962d198da874092 2024-11-07T12:43:48,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/9de6a23c0b334b6db88a00c9f7ac5aec as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/9de6a23c0b334b6db88a00c9f7ac5aec 2024-11-07T12:43:48,436 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/9de6a23c0b334b6db88a00c9f7ac5aec, entries=150, sequenceid=450, filesize=30.5 K 2024-11-07T12:43:48,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/20b308226f694491aea04304eb607ef5 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/20b308226f694491aea04304eb607ef5 2024-11-07T12:43:48,440 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/20b308226f694491aea04304eb607ef5, entries=150, sequenceid=450, filesize=12.0 K 2024-11-07T12:43:48,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/0a7639e03b3b43c78962d198da874092 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/0a7639e03b3b43c78962d198da874092 2024-11-07T12:43:48,445 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/0a7639e03b3b43c78962d198da874092, entries=150, sequenceid=450, filesize=12.0 K 2024-11-07T12:43:48,446 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 37eda6b0dad56caa0fa6c1ac8208d5b2 in 1383ms, sequenceid=450, compaction requested=false 2024-11-07T12:43:48,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegion(2603): Flush status journal for 37eda6b0dad56caa0fa6c1ac8208d5b2: 2024-11-07T12:43:48,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:48,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=179 2024-11-07T12:43:48,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4169): Remote procedure done, pid=179 2024-11-07T12:43:48,449 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=179, resume processing ppid=178 2024-11-07T12:43:48,449 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=179, ppid=178, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0010 sec 2024-11-07T12:43:48,451 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=178, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=178, table=TestAcidGuarantees in 2.0070 sec 2024-11-07T12:43:48,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-11-07T12:43:48,592 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:43:48,592 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:43:48,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] procedure2.ProcedureExecutor(1139): Stored pid=180, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=180, table=TestAcidGuarantees 2024-11-07T12:43:48,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=180 2024-11-07T12:43:48,594 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=180, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=180, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:43:48,595 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=180, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=180, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:43:48,595 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=181, ppid=180, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:43:48,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=180 2024-11-07T12:43:48,748 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=181 2024-11-07T12:43:48,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:48,748 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegion(2902): Flushing 37eda6b0dad56caa0fa6c1ac8208d5b2 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-07T12:43:48,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=A 2024-11-07T12:43:48,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:48,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=B 2024-11-07T12:43:48,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:48,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=C 2024-11-07T12:43:48,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:48,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107d5f74e69f07b4d3d908a06cde799ecf5_37eda6b0dad56caa0fa6c1ac8208d5b2 is 50, key is test_row_0/A:col10/1730983427223/Put/seqid=0 2024-11-07T12:43:48,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742616_1792 (size=12454) 2024-11-07T12:43:48,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=180 2024-11-07T12:43:49,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,215 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107d5f74e69f07b4d3d908a06cde799ecf5_37eda6b0dad56caa0fa6c1ac8208d5b2 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107d5f74e69f07b4d3d908a06cde799ecf5_37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:49,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/040d445c4b4d4d35b4ba9f3e6a6488fa, store: [table=TestAcidGuarantees family=A region=37eda6b0dad56caa0fa6c1ac8208d5b2] 2024-11-07T12:43:49,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/040d445c4b4d4d35b4ba9f3e6a6488fa is 175, key is test_row_0/A:col10/1730983427223/Put/seqid=0 2024-11-07T12:43:49,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=180 2024-11-07T12:43:49,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742617_1793 (size=31255) 2024-11-07T12:43:49,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:49,395 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. as already flushing 2024-11-07T12:43:49,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:49,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47946 deadline: 1730983489415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:49,421 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:49,421 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:49,421 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:49,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:49,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1730983489420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:49,426 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:49,426 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:49,426 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:49,533 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:49,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1730983489531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:49,534 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:49,534 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:49,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47946 deadline: 1730983489531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:49,534 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:49,534 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:49,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:49,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:49,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:49,631 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=473, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/040d445c4b4d4d35b4ba9f3e6a6488fa 2024-11-07T12:43:49,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/a68fbf4366ac44128e0bd3c16ea7ee8d is 50, key is test_row_0/B:col10/1730983427223/Put/seqid=0 2024-11-07T12:43:49,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742618_1794 (size=12301) 2024-11-07T12:43:49,642 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=473 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/a68fbf4366ac44128e0bd3c16ea7ee8d 2024-11-07T12:43:49,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/93f792033b0f4644a8b5c8fa3be56607 is 50, key is test_row_0/C:col10/1730983427223/Put/seqid=0 2024-11-07T12:43:49,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742619_1795 (size=12301) 2024-11-07T12:43:49,671 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=473 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/93f792033b0f4644a8b5c8fa3be56607 2024-11-07T12:43:49,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/040d445c4b4d4d35b4ba9f3e6a6488fa as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/040d445c4b4d4d35b4ba9f3e6a6488fa 2024-11-07T12:43:49,679 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/040d445c4b4d4d35b4ba9f3e6a6488fa, entries=150, sequenceid=473, filesize=30.5 K 2024-11-07T12:43:49,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/a68fbf4366ac44128e0bd3c16ea7ee8d as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/a68fbf4366ac44128e0bd3c16ea7ee8d 2024-11-07T12:43:49,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,684 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/a68fbf4366ac44128e0bd3c16ea7ee8d, entries=150, sequenceid=473, filesize=12.0 K 2024-11-07T12:43:49,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/93f792033b0f4644a8b5c8fa3be56607 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/93f792033b0f4644a8b5c8fa3be56607 2024-11-07T12:43:49,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,690 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/93f792033b0f4644a8b5c8fa3be56607, entries=150, sequenceid=473, filesize=12.0 K 2024-11-07T12:43:49,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,691 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 37eda6b0dad56caa0fa6c1ac8208d5b2 in 943ms, sequenceid=473, compaction requested=true 2024-11-07T12:43:49,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegion(2603): Flush status journal for 37eda6b0dad56caa0fa6c1ac8208d5b2: 2024-11-07T12:43:49,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:49,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=181 2024-11-07T12:43:49,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4169): Remote procedure done, pid=181 2024-11-07T12:43:49,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,695 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=181, resume processing ppid=180 2024-11-07T12:43:49,695 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=181, ppid=180, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0970 sec 2024-11-07T12:43:49,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,697 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=180, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=180, table=TestAcidGuarantees in 1.1040 sec 2024-11-07T12:43:49,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=180 2024-11-07T12:43:49,732 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:43:49,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,732 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:43:49,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] procedure2.ProcedureExecutor(1139): Stored pid=182, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=182, table=TestAcidGuarantees 2024-11-07T12:43:49,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=182 2024-11-07T12:43:49,734 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=182, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=182, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:43:49,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,735 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=182, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=182, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:43:49,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,735 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=183, ppid=182, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:43:49,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:49,748 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 37eda6b0dad56caa0fa6c1ac8208d5b2 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-07T12:43:49,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=A 2024-11-07T12:43:49,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:49,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=B 2024-11-07T12:43:49,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:49,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=C 2024-11-07T12:43:49,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:49,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,756 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107309382900c0a4be4b931692b438ee0aa_37eda6b0dad56caa0fa6c1ac8208d5b2 is 50, key is test_row_0/A:col10/1730983429748/Put/seqid=0 2024-11-07T12:43:49,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:49,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742620_1796 (size=14994) 2024-11-07T12:43:49,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:49,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1730983489805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:49,811 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:49,811 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:49,811 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:49,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:49,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47946 deadline: 1730983489811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:49,815 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:49,815 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:49,815 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:49,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=182 2024-11-07T12:43:49,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:49,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47962 deadline: 1730983489872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:49,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:49,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47926 deadline: 1730983489872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:49,876 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to db9ad1cb6cf9,34293,1730983260217 for 'test_row_2' in 37eda6b0dad56caa0fa6c1ac8208d5b2 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:43:49,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:49,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:49,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:49,876 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to db9ad1cb6cf9,34293,1730983260217 for 'test_row_0' in 37eda6b0dad56caa0fa6c1ac8208d5b2 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:43:49,877 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:49,877 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:49,877 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:49,888 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=183 2024-11-07T12:43:49,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:49,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. as already flushing 2024-11-07T12:43:49,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:49,888 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] handler.RSProcedureHandler(58): pid=183 java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:49,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=183 java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:49,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=183 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:49,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:49,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47946 deadline: 1730983489922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:49,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:49,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1730983489922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:49,927 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:49,927 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:49,927 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:49,928 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:49,928 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:49,928 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:50,041 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=183 2024-11-07T12:43:50,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:50,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. as already flushing 2024-11-07T12:43:50,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:50,042 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=183}] handler.RSProcedureHandler(58): pid=183 java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:50,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=183 java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:50,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=183 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:50,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=182 2024-11-07T12:43:50,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:50,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1730983490131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:50,137 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:50,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:50,137 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:50,137 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:50,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47946 deadline: 1730983490132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:50,137 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:50,138 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:50,138 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:50,175 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:50,180 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107309382900c0a4be4b931692b438ee0aa_37eda6b0dad56caa0fa6c1ac8208d5b2 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107309382900c0a4be4b931692b438ee0aa_37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:50,181 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/53c77c5f5dcc48daa33cac83f78f4558, store: [table=TestAcidGuarantees family=A region=37eda6b0dad56caa0fa6c1ac8208d5b2] 2024-11-07T12:43:50,181 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/53c77c5f5dcc48daa33cac83f78f4558 is 175, key is test_row_0/A:col10/1730983429748/Put/seqid=0 2024-11-07T12:43:50,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742621_1797 (size=39949) 2024-11-07T12:43:50,188 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=487, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/53c77c5f5dcc48daa33cac83f78f4558 2024-11-07T12:43:50,198 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=183 2024-11-07T12:43:50,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:50,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. as already flushing 2024-11-07T12:43:50,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:50,198 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=183}] handler.RSProcedureHandler(58): pid=183 java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:50,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=183 java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:50,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=183 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:50,210 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/b357060658cd41e38f55707835da2140 is 50, key is test_row_0/B:col10/1730983429748/Put/seqid=0 2024-11-07T12:43:50,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742622_1798 (size=12301) 2024-11-07T12:43:50,225 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=487 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/b357060658cd41e38f55707835da2140 2024-11-07T12:43:50,242 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/eb60687b20ee43e5a0b3522b44456222 is 50, key is test_row_0/C:col10/1730983429748/Put/seqid=0 2024-11-07T12:43:50,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742623_1799 (size=12301) 2024-11-07T12:43:50,264 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=487 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/eb60687b20ee43e5a0b3522b44456222 2024-11-07T12:43:50,267 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/53c77c5f5dcc48daa33cac83f78f4558 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/53c77c5f5dcc48daa33cac83f78f4558 2024-11-07T12:43:50,271 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/53c77c5f5dcc48daa33cac83f78f4558, entries=200, sequenceid=487, filesize=39.0 K 2024-11-07T12:43:50,273 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/b357060658cd41e38f55707835da2140 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/b357060658cd41e38f55707835da2140 2024-11-07T12:43:50,278 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/b357060658cd41e38f55707835da2140, entries=150, sequenceid=487, filesize=12.0 K 2024-11-07T12:43:50,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/eb60687b20ee43e5a0b3522b44456222 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/eb60687b20ee43e5a0b3522b44456222 2024-11-07T12:43:50,282 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/eb60687b20ee43e5a0b3522b44456222, entries=150, sequenceid=487, filesize=12.0 K 2024-11-07T12:43:50,283 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 37eda6b0dad56caa0fa6c1ac8208d5b2 in 535ms, sequenceid=487, compaction requested=true 2024-11-07T12:43:50,283 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 37eda6b0dad56caa0fa6c1ac8208d5b2: 2024-11-07T12:43:50,283 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:43:50,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 37eda6b0dad56caa0fa6c1ac8208d5b2:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:43:50,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:50,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 37eda6b0dad56caa0fa6c1ac8208d5b2:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:43:50,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:50,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 37eda6b0dad56caa0fa6c1ac8208d5b2:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:43:50,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:43:50,284 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:43:50,284 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134770 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:43:50,284 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): 37eda6b0dad56caa0fa6c1ac8208d5b2/A is initiating minor compaction (all files) 2024-11-07T12:43:50,284 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 37eda6b0dad56caa0fa6c1ac8208d5b2/A in TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:50,285 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50294 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:43:50,285 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/604166de2dcd4e6fa3831797fdcae72b, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/9de6a23c0b334b6db88a00c9f7ac5aec, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/040d445c4b4d4d35b4ba9f3e6a6488fa, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/53c77c5f5dcc48daa33cac83f78f4558] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp, totalSize=131.6 K 2024-11-07T12:43:50,285 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 37eda6b0dad56caa0fa6c1ac8208d5b2/B is initiating minor compaction (all files) 2024-11-07T12:43:50,285 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:50,285 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 37eda6b0dad56caa0fa6c1ac8208d5b2/B in TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:50,285 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. files: [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/604166de2dcd4e6fa3831797fdcae72b, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/9de6a23c0b334b6db88a00c9f7ac5aec, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/040d445c4b4d4d35b4ba9f3e6a6488fa, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/53c77c5f5dcc48daa33cac83f78f4558] 2024-11-07T12:43:50,285 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/0761afa32d974e6d984049e1e5e617a8, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/20b308226f694491aea04304eb607ef5, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/a68fbf4366ac44128e0bd3c16ea7ee8d, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/b357060658cd41e38f55707835da2140] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp, totalSize=49.1 K 2024-11-07T12:43:50,285 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 604166de2dcd4e6fa3831797fdcae72b, keycount=150, bloomtype=ROW, size=31.6 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1730983425679 2024-11-07T12:43:50,285 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 0761afa32d974e6d984049e1e5e617a8, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1730983425679 2024-11-07T12:43:50,286 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 20b308226f694491aea04304eb607ef5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=450, earliestPutTs=1730983426028 2024-11-07T12:43:50,286 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9de6a23c0b334b6db88a00c9f7ac5aec, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=450, earliestPutTs=1730983426028 2024-11-07T12:43:50,286 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting a68fbf4366ac44128e0bd3c16ea7ee8d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=473, earliestPutTs=1730983427216 2024-11-07T12:43:50,286 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 040d445c4b4d4d35b4ba9f3e6a6488fa, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=473, earliestPutTs=1730983427216 2024-11-07T12:43:50,287 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting b357060658cd41e38f55707835da2140, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=487, earliestPutTs=1730983429412 2024-11-07T12:43:50,287 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53c77c5f5dcc48daa33cac83f78f4558, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=487, earliestPutTs=1730983429412 2024-11-07T12:43:50,295 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=37eda6b0dad56caa0fa6c1ac8208d5b2] 2024-11-07T12:43:50,297 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241107f5c9343ae6ea40b9ba006ddfb9324835_37eda6b0dad56caa0fa6c1ac8208d5b2 store=[table=TestAcidGuarantees family=A region=37eda6b0dad56caa0fa6c1ac8208d5b2] 2024-11-07T12:43:50,298 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 37eda6b0dad56caa0fa6c1ac8208d5b2#B#compaction#672 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:43:50,298 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/f62c93813fc9469c9e6537c401e5afda is 50, key is test_row_0/B:col10/1730983429748/Put/seqid=0 2024-11-07T12:43:50,300 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241107f5c9343ae6ea40b9ba006ddfb9324835_37eda6b0dad56caa0fa6c1ac8208d5b2, store=[table=TestAcidGuarantees family=A region=37eda6b0dad56caa0fa6c1ac8208d5b2] 2024-11-07T12:43:50,300 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107f5c9343ae6ea40b9ba006ddfb9324835_37eda6b0dad56caa0fa6c1ac8208d5b2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=37eda6b0dad56caa0fa6c1ac8208d5b2] 2024-11-07T12:43:50,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742625_1801 (size=4469) 2024-11-07T12:43:50,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742624_1800 (size=13527) 2024-11-07T12:43:50,352 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=183 2024-11-07T12:43:50,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:50,353 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.HRegion(2902): Flushing 37eda6b0dad56caa0fa6c1ac8208d5b2 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-07T12:43:50,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=A 2024-11-07T12:43:50,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:50,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=B 2024-11-07T12:43:50,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:50,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=C 2024-11-07T12:43:50,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:50,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=182 2024-11-07T12:43:50,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411070113c49441a04189ae827673b54b0b45_37eda6b0dad56caa0fa6c1ac8208d5b2 is 50, key is test_row_0/A:col10/1730983429775/Put/seqid=0 2024-11-07T12:43:50,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742626_1802 (size=12454) 2024-11-07T12:43:50,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:50,385 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411070113c49441a04189ae827673b54b0b45_37eda6b0dad56caa0fa6c1ac8208d5b2 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411070113c49441a04189ae827673b54b0b45_37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:50,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/0234f3af39da4767a907c20fbe367967, store: [table=TestAcidGuarantees family=A region=37eda6b0dad56caa0fa6c1ac8208d5b2] 2024-11-07T12:43:50,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/0234f3af39da4767a907c20fbe367967 is 175, key is test_row_0/A:col10/1730983429775/Put/seqid=0 2024-11-07T12:43:50,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742627_1803 (size=31255) 2024-11-07T12:43:50,392 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=505, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/0234f3af39da4767a907c20fbe367967 2024-11-07T12:43:50,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/c8b9bc5f17f94aec8af0690635a56723 is 50, key is test_row_0/B:col10/1730983429775/Put/seqid=0 2024-11-07T12:43:50,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742628_1804 (size=12301) 2024-11-07T12:43:50,445 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=505 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/c8b9bc5f17f94aec8af0690635a56723 2024-11-07T12:43:50,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:50,446 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. as already flushing 2024-11-07T12:43:50,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/4bb818caaddb492d866e66f4a43312f4 is 50, key is test_row_0/C:col10/1730983429775/Put/seqid=0 2024-11-07T12:43:50,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742629_1805 (size=12301) 2024-11-07T12:43:50,489 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=505 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/4bb818caaddb492d866e66f4a43312f4 2024-11-07T12:43:50,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:50,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47946 deadline: 1730983490484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:50,490 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:50,490 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:50,490 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:50,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:50,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1730983490486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:50,494 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:50,494 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:50,494 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:50,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/0234f3af39da4767a907c20fbe367967 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/0234f3af39da4767a907c20fbe367967 2024-11-07T12:43:50,499 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/0234f3af39da4767a907c20fbe367967, entries=150, sequenceid=505, filesize=30.5 K 2024-11-07T12:43:50,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/c8b9bc5f17f94aec8af0690635a56723 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/c8b9bc5f17f94aec8af0690635a56723 2024-11-07T12:43:50,505 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/c8b9bc5f17f94aec8af0690635a56723, entries=150, sequenceid=505, filesize=12.0 K 2024-11-07T12:43:50,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/4bb818caaddb492d866e66f4a43312f4 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/4bb818caaddb492d866e66f4a43312f4 2024-11-07T12:43:50,509 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/4bb818caaddb492d866e66f4a43312f4, entries=150, sequenceid=505, filesize=12.0 K 2024-11-07T12:43:50,510 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 37eda6b0dad56caa0fa6c1ac8208d5b2 in 157ms, sequenceid=505, compaction requested=true 2024-11-07T12:43:50,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.HRegion(2603): Flush status journal for 37eda6b0dad56caa0fa6c1ac8208d5b2: 2024-11-07T12:43:50,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:50,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=183 2024-11-07T12:43:50,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4169): Remote procedure done, pid=183 2024-11-07T12:43:50,513 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=183, resume processing ppid=182 2024-11-07T12:43:50,513 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=183, ppid=182, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 776 msec 2024-11-07T12:43:50,515 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=182, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=182, table=TestAcidGuarantees in 782 msec 2024-11-07T12:43:50,597 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 37eda6b0dad56caa0fa6c1ac8208d5b2 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-07T12:43:50,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=A 2024-11-07T12:43:50,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:50,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=B 2024-11-07T12:43:50,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:50,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=C 2024-11-07T12:43:50,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:50,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:50,607 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107e59d21d435d043139bc144ce9d23b99f_37eda6b0dad56caa0fa6c1ac8208d5b2 is 50, key is test_row_0/A:col10/1730983430485/Put/seqid=0 2024-11-07T12:43:50,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742630_1806 (size=14994) 2024-11-07T12:43:50,628 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:50,642 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107e59d21d435d043139bc144ce9d23b99f_37eda6b0dad56caa0fa6c1ac8208d5b2 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107e59d21d435d043139bc144ce9d23b99f_37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:50,646 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/d945eb9f65964b37993c4e0f3e42c78c, store: [table=TestAcidGuarantees family=A region=37eda6b0dad56caa0fa6c1ac8208d5b2] 2024-11-07T12:43:50,647 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/d945eb9f65964b37993c4e0f3e42c78c is 175, key is test_row_0/A:col10/1730983430485/Put/seqid=0 2024-11-07T12:43:50,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742631_1807 (size=39949) 2024-11-07T12:43:50,671 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=520, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/d945eb9f65964b37993c4e0f3e42c78c 2024-11-07T12:43:50,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:50,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1730983490668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:50,678 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:50,678 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:50,678 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:50,686 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:50,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47946 deadline: 1730983490678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:50,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:50,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:50,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:50,688 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/c8a7f581f8584b5686574c7df9cefc5f is 50, key is test_row_0/B:col10/1730983430485/Put/seqid=0 2024-11-07T12:43:50,707 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 37eda6b0dad56caa0fa6c1ac8208d5b2#A#compaction#673 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:43:50,708 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/143b9d9a469c4f97b6df3c6a802b50d4 is 175, key is test_row_0/A:col10/1730983429748/Put/seqid=0 2024-11-07T12:43:50,718 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/f62c93813fc9469c9e6537c401e5afda as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/f62c93813fc9469c9e6537c401e5afda 2024-11-07T12:43:50,723 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 37eda6b0dad56caa0fa6c1ac8208d5b2/B of 37eda6b0dad56caa0fa6c1ac8208d5b2 into f62c93813fc9469c9e6537c401e5afda(size=13.2 K), total size for store is 25.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:43:50,723 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 37eda6b0dad56caa0fa6c1ac8208d5b2: 2024-11-07T12:43:50,723 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., storeName=37eda6b0dad56caa0fa6c1ac8208d5b2/B, priority=12, startTime=1730983430283; duration=0sec 2024-11-07T12:43:50,723 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:43:50,723 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 37eda6b0dad56caa0fa6c1ac8208d5b2:B 2024-11-07T12:43:50,723 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-07T12:43:50,727 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62595 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-07T12:43:50,727 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 37eda6b0dad56caa0fa6c1ac8208d5b2/C is initiating minor compaction (all files) 2024-11-07T12:43:50,727 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 37eda6b0dad56caa0fa6c1ac8208d5b2/C in TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:50,728 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/0fc56b1d51364298a349e0d2758ab3c3, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/0a7639e03b3b43c78962d198da874092, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/93f792033b0f4644a8b5c8fa3be56607, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/eb60687b20ee43e5a0b3522b44456222, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/4bb818caaddb492d866e66f4a43312f4] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp, totalSize=61.1 K 2024-11-07T12:43:50,728 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 0fc56b1d51364298a349e0d2758ab3c3, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1730983425679 2024-11-07T12:43:50,728 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a7639e03b3b43c78962d198da874092, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=450, earliestPutTs=1730983426028 2024-11-07T12:43:50,729 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 93f792033b0f4644a8b5c8fa3be56607, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=473, earliestPutTs=1730983427216 2024-11-07T12:43:50,729 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting eb60687b20ee43e5a0b3522b44456222, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=487, earliestPutTs=1730983429412 2024-11-07T12:43:50,729 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting 4bb818caaddb492d866e66f4a43312f4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=505, earliestPutTs=1730983429775 2024-11-07T12:43:50,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742632_1808 (size=12301) 2024-11-07T12:43:50,747 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=520 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/c8a7f581f8584b5686574c7df9cefc5f 2024-11-07T12:43:50,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742633_1809 (size=32447) 2024-11-07T12:43:50,770 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/143b9d9a469c4f97b6df3c6a802b50d4 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/143b9d9a469c4f97b6df3c6a802b50d4 2024-11-07T12:43:50,770 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 37eda6b0dad56caa0fa6c1ac8208d5b2#C#compaction#679 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:43:50,771 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/522e4a1167f847d09959a8390277076a is 50, key is test_row_0/C:col10/1730983429775/Put/seqid=0 2024-11-07T12:43:50,774 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/f4286612e2cc4644b0a26b72e841037d is 50, key is test_row_0/C:col10/1730983430485/Put/seqid=0 2024-11-07T12:43:50,776 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 37eda6b0dad56caa0fa6c1ac8208d5b2/A of 37eda6b0dad56caa0fa6c1ac8208d5b2 into 143b9d9a469c4f97b6df3c6a802b50d4(size=31.7 K), total size for store is 62.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:43:50,777 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 37eda6b0dad56caa0fa6c1ac8208d5b2: 2024-11-07T12:43:50,777 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., storeName=37eda6b0dad56caa0fa6c1ac8208d5b2/A, priority=12, startTime=1730983430283; duration=0sec 2024-11-07T12:43:50,777 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:50,777 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 37eda6b0dad56caa0fa6c1ac8208d5b2:A 2024-11-07T12:43:50,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742634_1810 (size=13561) 2024-11-07T12:43:50,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:50,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1730983490782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:50,789 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:50,789 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:50,789 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:50,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:50,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47946 deadline: 1730983490791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:50,796 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:50,796 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:50,796 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:50,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742635_1811 (size=12301) 2024-11-07T12:43:50,798 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=520 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/f4286612e2cc4644b0a26b72e841037d 2024-11-07T12:43:50,803 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/d945eb9f65964b37993c4e0f3e42c78c as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/d945eb9f65964b37993c4e0f3e42c78c 2024-11-07T12:43:50,807 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/d945eb9f65964b37993c4e0f3e42c78c, entries=200, sequenceid=520, filesize=39.0 K 2024-11-07T12:43:50,807 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/c8a7f581f8584b5686574c7df9cefc5f as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/c8a7f581f8584b5686574c7df9cefc5f 2024-11-07T12:43:50,812 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/c8a7f581f8584b5686574c7df9cefc5f, entries=150, sequenceid=520, filesize=12.0 K 2024-11-07T12:43:50,813 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/f4286612e2cc4644b0a26b72e841037d as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/f4286612e2cc4644b0a26b72e841037d 2024-11-07T12:43:50,816 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/f4286612e2cc4644b0a26b72e841037d, entries=150, sequenceid=520, filesize=12.0 K 2024-11-07T12:43:50,817 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 37eda6b0dad56caa0fa6c1ac8208d5b2 in 220ms, sequenceid=520, compaction requested=true 2024-11-07T12:43:50,817 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 37eda6b0dad56caa0fa6c1ac8208d5b2: 2024-11-07T12:43:50,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 37eda6b0dad56caa0fa6c1ac8208d5b2:A, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:43:50,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:50,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 37eda6b0dad56caa0fa6c1ac8208d5b2:B, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:43:50,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:43:50,817 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:43:50,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 37eda6b0dad56caa0fa6c1ac8208d5b2:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:43:50,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-07T12:43:50,820 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103651 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:43:50,820 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): 37eda6b0dad56caa0fa6c1ac8208d5b2/A is initiating minor compaction (all files) 2024-11-07T12:43:50,820 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 37eda6b0dad56caa0fa6c1ac8208d5b2/A in TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:50,820 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/143b9d9a469c4f97b6df3c6a802b50d4, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/0234f3af39da4767a907c20fbe367967, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/d945eb9f65964b37993c4e0f3e42c78c] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp, totalSize=101.2 K 2024-11-07T12:43:50,820 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:50,820 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. files: [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/143b9d9a469c4f97b6df3c6a802b50d4, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/0234f3af39da4767a907c20fbe367967, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/d945eb9f65964b37993c4e0f3e42c78c] 2024-11-07T12:43:50,821 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 143b9d9a469c4f97b6df3c6a802b50d4, keycount=150, bloomtype=ROW, size=31.7 K, encoding=NONE, compression=NONE, seqNum=487, earliestPutTs=1730983429412 2024-11-07T12:43:50,822 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0234f3af39da4767a907c20fbe367967, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=505, earliestPutTs=1730983429775 2024-11-07T12:43:50,822 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting d945eb9f65964b37993c4e0f3e42c78c, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=520, earliestPutTs=1730983430475 2024-11-07T12:43:50,845 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=37eda6b0dad56caa0fa6c1ac8208d5b2] 2024-11-07T12:43:50,853 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241107c26b6d0201d7401daef6544a90a9777f_37eda6b0dad56caa0fa6c1ac8208d5b2 store=[table=TestAcidGuarantees family=A region=37eda6b0dad56caa0fa6c1ac8208d5b2] 2024-11-07T12:43:50,855 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241107c26b6d0201d7401daef6544a90a9777f_37eda6b0dad56caa0fa6c1ac8208d5b2, store=[table=TestAcidGuarantees family=A region=37eda6b0dad56caa0fa6c1ac8208d5b2] 2024-11-07T12:43:50,855 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107c26b6d0201d7401daef6544a90a9777f_37eda6b0dad56caa0fa6c1ac8208d5b2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=37eda6b0dad56caa0fa6c1ac8208d5b2] 2024-11-07T12:43:50,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=182 2024-11-07T12:43:50,873 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:43:50,874 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:43:50,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] procedure2.ProcedureExecutor(1139): Stored pid=184, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=184, table=TestAcidGuarantees 2024-11-07T12:43:50,876 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=184, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=184, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:43:50,877 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=184, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=184, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:43:50,877 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=185, ppid=184, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:43:50,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=184 2024-11-07T12:43:50,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742636_1812 (size=4469) 2024-11-07T12:43:50,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=184 2024-11-07T12:43:51,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:51,001 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 37eda6b0dad56caa0fa6c1ac8208d5b2 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-07T12:43:51,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=A 2024-11-07T12:43:51,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:51,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=B 2024-11-07T12:43:51,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:51,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=C 2024-11-07T12:43:51,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:51,016 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107e17778132d1f4d629b28402a778d0677_37eda6b0dad56caa0fa6c1ac8208d5b2 is 50, key is test_row_0/A:col10/1730983430671/Put/seqid=0 2024-11-07T12:43:51,030 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=185 2024-11-07T12:43:51,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:51,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. as already flushing 2024-11-07T12:43:51,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:51,030 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] handler.RSProcedureHandler(58): pid=185 java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:51,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=185 java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:51,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742637_1813 (size=14994) 2024-11-07T12:43:51,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=185 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:51,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:51,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47946 deadline: 1730983491046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:51,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:51,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:51,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:51,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:51,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1730983491051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:51,060 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:51,060 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:51,060 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:51,136 INFO [Thread-2895 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2895. 2024-11-07T12:43:51,136 DEBUG [Thread-2895 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:43:51,136 DEBUG [Thread-2895 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:43:51,136 INFO [Thread-2889 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2889. 2024-11-07T12:43:51,136 DEBUG [Thread-2889 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:43:51,136 DEBUG [Thread-2889 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:43:51,137 DEBUG [Thread-2889 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:43:51,137 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:43:51,137 DEBUG [Thread-2895 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:43:51,137 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:43:51,139 INFO [Thread-2893 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2893. 2024-11-07T12:43:51,139 DEBUG [Thread-2893 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:43:51,139 DEBUG [Thread-2893 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:43:51,139 DEBUG [Thread-2893 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:43:51,139 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:43:51,141 INFO [Thread-2891 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2891. 2024-11-07T12:43:51,141 DEBUG [Thread-2891 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:43:51,141 DEBUG [Thread-2891 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:43:51,141 DEBUG [Thread-2891 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:43:51,141 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:43:51,142 INFO [Thread-2887 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2887. 2024-11-07T12:43:51,142 DEBUG [Thread-2887 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:43:51,142 DEBUG [Thread-2887 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:43:51,143 DEBUG [Thread-2887 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:43:51,143 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:43:51,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:51,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47946 deadline: 1730983491162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:51,162 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:51,162 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:51,162 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:51,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:51,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1730983491172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:51,172 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:51,172 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:51,172 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:51,182 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=185 2024-11-07T12:43:51,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:51,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. as already flushing 2024-11-07T12:43:51,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:51,183 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=185}] handler.RSProcedureHandler(58): pid=185 java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:51,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=185 java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:51,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=185 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:51,192 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/522e4a1167f847d09959a8390277076a as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/522e4a1167f847d09959a8390277076a 2024-11-07T12:43:51,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=184 2024-11-07T12:43:51,197 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 37eda6b0dad56caa0fa6c1ac8208d5b2/C of 37eda6b0dad56caa0fa6c1ac8208d5b2 into 522e4a1167f847d09959a8390277076a(size=13.2 K), total size for store is 25.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:43:51,197 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 37eda6b0dad56caa0fa6c1ac8208d5b2: 2024-11-07T12:43:51,197 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., storeName=37eda6b0dad56caa0fa6c1ac8208d5b2/C, priority=11, startTime=1730983430284; duration=0sec 2024-11-07T12:43:51,197 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-07T12:43:51,197 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 37eda6b0dad56caa0fa6c1ac8208d5b2:C 2024-11-07T12:43:51,197 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 37eda6b0dad56caa0fa6c1ac8208d5b2:C 2024-11-07T12:43:51,197 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:43:51,198 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38129 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:43:51,198 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1541): 37eda6b0dad56caa0fa6c1ac8208d5b2/B is initiating minor compaction (all files) 2024-11-07T12:43:51,198 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 37eda6b0dad56caa0fa6c1ac8208d5b2/B in TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:51,198 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/f62c93813fc9469c9e6537c401e5afda, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/c8b9bc5f17f94aec8af0690635a56723, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/c8a7f581f8584b5686574c7df9cefc5f] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp, totalSize=37.2 K 2024-11-07T12:43:51,199 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting f62c93813fc9469c9e6537c401e5afda, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=487, earliestPutTs=1730983429412 2024-11-07T12:43:51,199 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting c8b9bc5f17f94aec8af0690635a56723, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=505, earliestPutTs=1730983429775 2024-11-07T12:43:51,199 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.Compactor(225): Compacting c8a7f581f8584b5686574c7df9cefc5f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=520, earliestPutTs=1730983430475 2024-11-07T12:43:51,211 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 37eda6b0dad56caa0fa6c1ac8208d5b2#B#compaction#683 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:43:51,212 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/9ea66c25743d415783583b4ca5d5be41 is 50, key is test_row_0/B:col10/1730983430485/Put/seqid=0 2024-11-07T12:43:51,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742638_1814 (size=13629) 2024-11-07T12:43:51,295 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 37eda6b0dad56caa0fa6c1ac8208d5b2#A#compaction#681 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:43:51,296 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/cb3064520f7b42d6ade3cb2186c1c18e is 175, key is test_row_0/A:col10/1730983430485/Put/seqid=0 2024-11-07T12:43:51,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742639_1815 (size=32549) 2024-11-07T12:43:51,336 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=185 2024-11-07T12:43:51,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:51,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. as already flushing 2024-11-07T12:43:51,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:51,337 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] handler.RSProcedureHandler(58): pid=185 java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:51,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=185 java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:51,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=185 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:51,372 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:51,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47946 deadline: 1730983491372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:51,372 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:51,372 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:51,373 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:51,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:51,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1730983491382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:51,383 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:51,383 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:51,383 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:51,432 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:51,435 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107e17778132d1f4d629b28402a778d0677_37eda6b0dad56caa0fa6c1ac8208d5b2 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107e17778132d1f4d629b28402a778d0677_37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:51,436 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/0503931b799e40858c9dd08fa5fe1184, store: [table=TestAcidGuarantees family=A region=37eda6b0dad56caa0fa6c1ac8208d5b2] 2024-11-07T12:43:51,436 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/0503931b799e40858c9dd08fa5fe1184 is 175, key is test_row_0/A:col10/1730983430671/Put/seqid=0 2024-11-07T12:43:51,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742640_1816 (size=39949) 2024-11-07T12:43:51,489 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=185 2024-11-07T12:43:51,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:51,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. as already flushing 2024-11-07T12:43:51,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:51,490 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] handler.RSProcedureHandler(58): pid=185 java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:51,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=185 java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:51,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=185 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:51,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=184 2024-11-07T12:43:51,620 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/9ea66c25743d415783583b4ca5d5be41 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/9ea66c25743d415783583b4ca5d5be41 2024-11-07T12:43:51,623 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 37eda6b0dad56caa0fa6c1ac8208d5b2/B of 37eda6b0dad56caa0fa6c1ac8208d5b2 into 9ea66c25743d415783583b4ca5d5be41(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:43:51,623 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 37eda6b0dad56caa0fa6c1ac8208d5b2: 2024-11-07T12:43:51,623 INFO [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., storeName=37eda6b0dad56caa0fa6c1ac8208d5b2/B, priority=13, startTime=1730983430817; duration=0sec 2024-11-07T12:43:51,623 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:43:51,623 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 37eda6b0dad56caa0fa6c1ac8208d5b2:B 2024-11-07T12:43:51,623 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-07T12:43:51,624 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-07T12:43:51,624 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-07T12:43:51,624 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. because compaction request was cancelled 2024-11-07T12:43:51,624 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 37eda6b0dad56caa0fa6c1ac8208d5b2:C 2024-11-07T12:43:51,642 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=185 2024-11-07T12:43:51,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:51,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. as already flushing 2024-11-07T12:43:51,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:51,643 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=185}] handler.RSProcedureHandler(58): pid=185 java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:51,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=185 java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:51,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=185 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:51,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:51,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47946 deadline: 1730983491681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:51,682 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:51,682 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:51,682 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:51,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:51,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1730983491692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:51,692 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:51,692 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:51,692 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:51,703 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/cb3064520f7b42d6ade3cb2186c1c18e as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/cb3064520f7b42d6ade3cb2186c1c18e 2024-11-07T12:43:51,706 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 37eda6b0dad56caa0fa6c1ac8208d5b2/A of 37eda6b0dad56caa0fa6c1ac8208d5b2 into cb3064520f7b42d6ade3cb2186c1c18e(size=31.8 K), total size for store is 31.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:43:51,706 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 37eda6b0dad56caa0fa6c1ac8208d5b2: 2024-11-07T12:43:51,706 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., storeName=37eda6b0dad56caa0fa6c1ac8208d5b2/A, priority=13, startTime=1730983430817; duration=0sec 2024-11-07T12:43:51,706 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:51,706 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 37eda6b0dad56caa0fa6c1ac8208d5b2:A 2024-11-07T12:43:51,795 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=185 2024-11-07T12:43:51,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:51,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. as already flushing 2024-11-07T12:43:51,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:51,795 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] handler.RSProcedureHandler(58): pid=185 java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:51,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=185 java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:51,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=185 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:51,839 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=540, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/0503931b799e40858c9dd08fa5fe1184 2024-11-07T12:43:51,844 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/d74b76bfdf1441ffaa8fc792c68bb3dc is 50, key is test_row_0/B:col10/1730983430671/Put/seqid=0 2024-11-07T12:43:51,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742641_1817 (size=12301) 2024-11-07T12:43:51,948 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=185 2024-11-07T12:43:51,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:51,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. as already flushing 2024-11-07T12:43:51,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:51,948 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] handler.RSProcedureHandler(58): pid=185 java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:51,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=185 java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:51,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=185 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:52,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=184 2024-11-07T12:43:52,100 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=185 2024-11-07T12:43:52,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:52,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. as already flushing 2024-11-07T12:43:52,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:52,101 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=185}] handler.RSProcedureHandler(58): pid=185 java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:52,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=185 java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:52,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=185 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:52,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:52,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47946 deadline: 1730983492191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:52,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:52,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:52,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:52,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:43:52,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1730983492201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:52,202 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 , the old value is region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:52,202 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:43:52,202 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., hostname=db9ad1cb6cf9,34293,1730983260217, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:43:52,247 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=540 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/d74b76bfdf1441ffaa8fc792c68bb3dc 2024-11-07T12:43:52,252 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/c3d2aa670a684b68a69fb2bb44076209 is 50, key is test_row_0/C:col10/1730983430671/Put/seqid=0 2024-11-07T12:43:52,253 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=185 2024-11-07T12:43:52,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:52,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. as already flushing 2024-11-07T12:43:52,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:52,254 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] handler.RSProcedureHandler(58): pid=185 java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:52,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=185 java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:52,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=185 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:52,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742642_1818 (size=12301) 2024-11-07T12:43:52,406 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=185 2024-11-07T12:43:52,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:52,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. as already flushing 2024-11-07T12:43:52,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:52,407 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] handler.RSProcedureHandler(58): pid=185 java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:52,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=185 java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:52,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=185 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:52,559 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=185 2024-11-07T12:43:52,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:52,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. as already flushing 2024-11-07T12:43:52,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:52,560 ERROR [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=185}] handler.RSProcedureHandler(58): pid=185 java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:52,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-0 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=185 java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:52,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4177): Remote procedure failed, pid=185 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:43:52,656 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=540 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/c3d2aa670a684b68a69fb2bb44076209 2024-11-07T12:43:52,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/0503931b799e40858c9dd08fa5fe1184 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/0503931b799e40858c9dd08fa5fe1184 2024-11-07T12:43:52,663 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/0503931b799e40858c9dd08fa5fe1184, entries=200, sequenceid=540, filesize=39.0 K 2024-11-07T12:43:52,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/d74b76bfdf1441ffaa8fc792c68bb3dc as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/d74b76bfdf1441ffaa8fc792c68bb3dc 2024-11-07T12:43:52,666 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/d74b76bfdf1441ffaa8fc792c68bb3dc, entries=150, sequenceid=540, filesize=12.0 K 2024-11-07T12:43:52,666 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/c3d2aa670a684b68a69fb2bb44076209 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/c3d2aa670a684b68a69fb2bb44076209 2024-11-07T12:43:52,669 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/c3d2aa670a684b68a69fb2bb44076209, entries=150, sequenceid=540, filesize=12.0 K 2024-11-07T12:43:52,670 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 37eda6b0dad56caa0fa6c1ac8208d5b2 in 1669ms, sequenceid=540, compaction requested=true 2024-11-07T12:43:52,670 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 37eda6b0dad56caa0fa6c1ac8208d5b2: 2024-11-07T12:43:52,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 37eda6b0dad56caa0fa6c1ac8208d5b2:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:43:52,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:52,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 37eda6b0dad56caa0fa6c1ac8208d5b2:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:43:52,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:52,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 37eda6b0dad56caa0fa6c1ac8208d5b2:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:43:52,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:43:52,670 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-07T12:43:52,670 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-07T12:43:52,671 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-07T12:43:52,671 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-07T12:43:52,671 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-07T12:43:52,671 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. because compaction request was cancelled 2024-11-07T12:43:52,671 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-07T12:43:52,671 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 37eda6b0dad56caa0fa6c1ac8208d5b2:A 2024-11-07T12:43:52,671 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:43:52,671 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. because compaction request was cancelled 2024-11-07T12:43:52,671 DEBUG [RS:0;db9ad1cb6cf9:34293-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 37eda6b0dad56caa0fa6c1ac8208d5b2:B 2024-11-07T12:43:52,671 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38163 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:43:52,671 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1541): 37eda6b0dad56caa0fa6c1ac8208d5b2/C is initiating minor compaction (all files) 2024-11-07T12:43:52,672 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 37eda6b0dad56caa0fa6c1ac8208d5b2/C in TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:52,672 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/522e4a1167f847d09959a8390277076a, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/f4286612e2cc4644b0a26b72e841037d, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/c3d2aa670a684b68a69fb2bb44076209] into tmpdir=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp, totalSize=37.3 K 2024-11-07T12:43:52,672 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting 522e4a1167f847d09959a8390277076a, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=505, earliestPutTs=1730983429775 2024-11-07T12:43:52,672 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4286612e2cc4644b0a26b72e841037d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=520, earliestPutTs=1730983430475 2024-11-07T12:43:52,672 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3d2aa670a684b68a69fb2bb44076209, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=540, earliestPutTs=1730983430656 2024-11-07T12:43:52,678 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 37eda6b0dad56caa0fa6c1ac8208d5b2#C#compaction#686 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:43:52,679 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/f5d1e5e0cd054afe99950343f5093a81 is 50, key is test_row_0/C:col10/1730983430671/Put/seqid=0 2024-11-07T12:43:52,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742643_1819 (size=13663) 2024-11-07T12:43:52,686 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/f5d1e5e0cd054afe99950343f5093a81 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/f5d1e5e0cd054afe99950343f5093a81 2024-11-07T12:43:52,689 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 37eda6b0dad56caa0fa6c1ac8208d5b2/C of 37eda6b0dad56caa0fa6c1ac8208d5b2 into f5d1e5e0cd054afe99950343f5093a81(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:43:52,689 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 37eda6b0dad56caa0fa6c1ac8208d5b2: 2024-11-07T12:43:52,689 INFO [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., storeName=37eda6b0dad56caa0fa6c1ac8208d5b2/C, priority=13, startTime=1730983432670; duration=0sec 2024-11-07T12:43:52,689 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:43:52,690 DEBUG [RS:0;db9ad1cb6cf9:34293-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 37eda6b0dad56caa0fa6c1ac8208d5b2:C 2024-11-07T12:43:52,712 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=185 2024-11-07T12:43:52,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:52,713 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.HRegion(2902): Flushing 37eda6b0dad56caa0fa6c1ac8208d5b2 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-07T12:43:52,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=A 2024-11-07T12:43:52,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:52,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=B 2024-11-07T12:43:52,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:52,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=C 2024-11-07T12:43:52,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:52,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411073dd10d30d9de48a79a15f71ea471f970_37eda6b0dad56caa0fa6c1ac8208d5b2 is 50, key is test_row_0/A:col10/1730983431043/Put/seqid=0 2024-11-07T12:43:52,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742644_1820 (size=12454) 2024-11-07T12:43:53,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=184 2024-11-07T12:43:53,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:53,124 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411073dd10d30d9de48a79a15f71ea471f970_37eda6b0dad56caa0fa6c1ac8208d5b2 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411073dd10d30d9de48a79a15f71ea471f970_37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:53,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/a97d15783a1b438fbe9fea2030fd64a0, store: [table=TestAcidGuarantees family=A region=37eda6b0dad56caa0fa6c1ac8208d5b2] 2024-11-07T12:43:53,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/a97d15783a1b438fbe9fea2030fd64a0 is 175, key is test_row_0/A:col10/1730983431043/Put/seqid=0 2024-11-07T12:43:53,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742645_1821 (size=31255) 2024-11-07T12:43:53,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34293 {}] regionserver.HRegion(8855): Flush requested on 37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:53,203 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. as already flushing 2024-11-07T12:43:53,203 INFO [Thread-2882 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2882. 2024-11-07T12:43:53,203 DEBUG [Thread-2882 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:43:53,203 DEBUG [Thread-2882 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:43:53,204 DEBUG [Thread-2882 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:43:53,204 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:43:53,213 INFO [Thread-2878 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2878. 2024-11-07T12:43:53,213 DEBUG [Thread-2878 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:43:53,213 DEBUG [Thread-2878 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:43:53,214 DEBUG [Thread-2878 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:43:53,214 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:43:53,529 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=558, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/a97d15783a1b438fbe9fea2030fd64a0 2024-11-07T12:43:53,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/e8d1a018faf141e997ca463c38f363c1 is 50, key is test_row_0/B:col10/1730983431043/Put/seqid=0 2024-11-07T12:43:53,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742646_1822 (size=12301) 2024-11-07T12:43:53,903 INFO [Thread-2876 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2876. 2024-11-07T12:43:53,903 DEBUG [Thread-2876 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:43:53,903 DEBUG [Thread-2876 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:43:53,903 DEBUG [Thread-2876 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:43:53,903 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:43:53,912 INFO [Thread-2880 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2880. 2024-11-07T12:43:53,912 DEBUG [Thread-2880 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:43:53,912 DEBUG [Thread-2880 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:43:53,912 DEBUG [Thread-2880 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:43:53,912 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:43:53,939 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=558 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/e8d1a018faf141e997ca463c38f363c1 2024-11-07T12:43:53,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/d3964a06491647979c15212fffebb987 is 50, key is test_row_0/C:col10/1730983431043/Put/seqid=0 2024-11-07T12:43:53,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742647_1823 (size=12301) 2024-11-07T12:43:54,349 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=558 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/d3964a06491647979c15212fffebb987 2024-11-07T12:43:54,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/a97d15783a1b438fbe9fea2030fd64a0 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/a97d15783a1b438fbe9fea2030fd64a0 2024-11-07T12:43:54,355 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/a97d15783a1b438fbe9fea2030fd64a0, entries=150, sequenceid=558, filesize=30.5 K 2024-11-07T12:43:54,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/e8d1a018faf141e997ca463c38f363c1 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/e8d1a018faf141e997ca463c38f363c1 2024-11-07T12:43:54,358 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/e8d1a018faf141e997ca463c38f363c1, entries=150, sequenceid=558, filesize=12.0 K 2024-11-07T12:43:54,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/d3964a06491647979c15212fffebb987 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/d3964a06491647979c15212fffebb987 2024-11-07T12:43:54,361 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/d3964a06491647979c15212fffebb987, entries=150, sequenceid=558, filesize=12.0 K 2024-11-07T12:43:54,362 INFO [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=26.84 KB/27480 for 37eda6b0dad56caa0fa6c1ac8208d5b2 in 1649ms, sequenceid=558, compaction requested=true 2024-11-07T12:43:54,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.HRegion(2603): Flush status journal for 37eda6b0dad56caa0fa6c1ac8208d5b2: 2024-11-07T12:43:54,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:54,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/db9ad1cb6cf9:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=185 2024-11-07T12:43:54,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster(4169): Remote procedure done, pid=185 2024-11-07T12:43:54,364 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=185, resume processing ppid=184 2024-11-07T12:43:54,364 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=185, ppid=184, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.4860 sec 2024-11-07T12:43:54,365 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=184, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=184, table=TestAcidGuarantees in 3.4910 sec 2024-11-07T12:43:55,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=184 2024-11-07T12:43:55,032 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:43:57,443 INFO [Thread-2884 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2884. 2024-11-07T12:43:57,443 DEBUG [Thread-2884 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:43:57,444 DEBUG [Thread-2884 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:43:57,444 DEBUG [Thread-2884 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:43:57,444 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:43:57,444 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-07T12:43:57,444 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 69 2024-11-07T12:43:57,444 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 123 2024-11-07T12:43:57,444 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 73 2024-11-07T12:43:57,444 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 117 2024-11-07T12:43:57,444 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 52 2024-11-07T12:43:57,444 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-07T12:43:57,444 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2483 2024-11-07T12:43:57,444 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2526 2024-11-07T12:43:57,444 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2499 2024-11-07T12:43:57,444 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2470 2024-11-07T12:43:57,444 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2517 2024-11-07T12:43:57,444 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-07T12:43:57,444 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-07T12:43:57,444 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T12:43:57,444 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:43:57,444 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:43:57,444 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:43:57,445 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-07T12:43:57,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] procedure2.ProcedureExecutor(1139): Stored pid=186, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-07T12:43:57,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=186 2024-11-07T12:43:57,448 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730983437447"}]},"ts":"1730983437447"} 2024-11-07T12:43:57,449 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-07T12:43:57,449 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-07T12:43:57,449 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=187, ppid=186, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-07T12:43:57,450 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=188, ppid=187, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=37eda6b0dad56caa0fa6c1ac8208d5b2, UNASSIGN}] 2024-11-07T12:43:57,451 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=188, ppid=187, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=37eda6b0dad56caa0fa6c1ac8208d5b2, UNASSIGN 2024-11-07T12:43:57,452 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=188 updating hbase:meta row=37eda6b0dad56caa0fa6c1ac8208d5b2, regionState=CLOSING, regionLocation=db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:57,453 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=188, ppid=187, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=37eda6b0dad56caa0fa6c1ac8208d5b2, UNASSIGN because future has completed 2024-11-07T12:43:57,453 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-07T12:43:57,453 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=189, ppid=188, state=RUNNABLE, hasLock=false; CloseRegionProcedure 37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217}] 2024-11-07T12:43:57,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=186 2024-11-07T12:43:57,605 INFO [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=189}] handler.UnassignRegionHandler(122): Close 37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:57,605 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=189}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-07T12:43:57,605 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=189}] regionserver.HRegion(1722): Closing 37eda6b0dad56caa0fa6c1ac8208d5b2, disabling compactions & flushes 2024-11-07T12:43:57,605 INFO [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=189}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:57,605 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=189}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:57,605 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=189}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. after waiting 0 ms 2024-11-07T12:43:57,605 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=189}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:57,605 INFO [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=189}] regionserver.HRegion(2902): Flushing 37eda6b0dad56caa0fa6c1ac8208d5b2 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-07T12:43:57,605 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=189}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=A 2024-11-07T12:43:57,606 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=189}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:57,606 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=189}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=B 2024-11-07T12:43:57,606 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=189}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:57,606 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=189}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37eda6b0dad56caa0fa6c1ac8208d5b2, store=C 2024-11-07T12:43:57,606 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=189}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-07T12:43:57,611 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=189}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110798adc6dfb99046f08cfda5cf6dff8203_37eda6b0dad56caa0fa6c1ac8208d5b2 is 50, key is test_row_0/A:col10/1730983433911/Put/seqid=0 2024-11-07T12:43:57,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742648_1824 (size=12454) 2024-11-07T12:43:57,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=186 2024-11-07T12:43:57,837 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-07T12:43:58,014 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=189}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:43:58,017 INFO [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=189}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110798adc6dfb99046f08cfda5cf6dff8203_37eda6b0dad56caa0fa6c1ac8208d5b2 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110798adc6dfb99046f08cfda5cf6dff8203_37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:58,018 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=189}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/485dff2dbbf5467f8f34f85d2ee17d5d, store: [table=TestAcidGuarantees family=A region=37eda6b0dad56caa0fa6c1ac8208d5b2] 2024-11-07T12:43:58,018 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=189}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/485dff2dbbf5467f8f34f85d2ee17d5d is 175, key is test_row_0/A:col10/1730983433911/Put/seqid=0 2024-11-07T12:43:58,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742649_1825 (size=31255) 2024-11-07T12:43:58,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=186 2024-11-07T12:43:58,421 INFO [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=189}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=566, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/485dff2dbbf5467f8f34f85d2ee17d5d 2024-11-07T12:43:58,426 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=189}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/e9241c374f6d48ea94772e4a8d218719 is 50, key is test_row_0/B:col10/1730983433911/Put/seqid=0 2024-11-07T12:43:58,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742650_1826 (size=12301) 2024-11-07T12:43:58,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=186 2024-11-07T12:43:58,829 INFO [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=189}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=566 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/e9241c374f6d48ea94772e4a8d218719 2024-11-07T12:43:58,834 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=189}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/b783d2f36e3142c9bb8164a5d00a1c9a is 50, key is test_row_0/C:col10/1730983433911/Put/seqid=0 2024-11-07T12:43:58,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742651_1827 (size=12301) 2024-11-07T12:43:59,237 INFO [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=189}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=566 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/b783d2f36e3142c9bb8164a5d00a1c9a 2024-11-07T12:43:59,241 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=189}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/A/485dff2dbbf5467f8f34f85d2ee17d5d as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/485dff2dbbf5467f8f34f85d2ee17d5d 2024-11-07T12:43:59,243 INFO [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=189}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/485dff2dbbf5467f8f34f85d2ee17d5d, entries=150, sequenceid=566, filesize=30.5 K 2024-11-07T12:43:59,244 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=189}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/B/e9241c374f6d48ea94772e4a8d218719 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/e9241c374f6d48ea94772e4a8d218719 2024-11-07T12:43:59,246 INFO [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=189}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/e9241c374f6d48ea94772e4a8d218719, entries=150, sequenceid=566, filesize=12.0 K 2024-11-07T12:43:59,247 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=189}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/.tmp/C/b783d2f36e3142c9bb8164a5d00a1c9a as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/b783d2f36e3142c9bb8164a5d00a1c9a 2024-11-07T12:43:59,249 INFO [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=189}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/b783d2f36e3142c9bb8164a5d00a1c9a, entries=150, sequenceid=566, filesize=12.0 K 2024-11-07T12:43:59,250 INFO [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=189}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 37eda6b0dad56caa0fa6c1ac8208d5b2 in 1645ms, sequenceid=566, compaction requested=true 2024-11-07T12:43:59,250 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/c7dee467bb744760bc636cc8a9f6ffb4, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/09780355c6714c6e9a59d31ee86f729c, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/8e40ed468304427499086b35a1d5d70d, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/c8022b066d3e49fcada23459efbcf950, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/308142f3672b41cab2eeac0a6eecef38, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/ac18115502bd40fab50c1f0a689a6759, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/43d6ff7d297e4a8fa7ef199c86972bf2, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/ff11c6e977764ce585ca73a34b014061, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/18951ee79df04451be85a8704467ede8, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/60b7b3d60b3d41fa9914eaf7b178f683, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/ad853594617743f9b3f8ccc4d4633d23, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/656d2e3dca7a4df2ac13fce49e35dc97, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/14da15b5f9be4df6827c69aa5b237cb2, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/40499930c9d44168bcb645af50ac9dfa, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/b9bcc05e42fb438f84da4b0fc99cda02, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/c0a81589ffca484197b5a1c574e0132e, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/61c15cd61ec44297b2d8b03b23d0a19c, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/f48af124758a4aebb4b22549fe26f479, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/7fac4daa648d4ac9be32b290b192a61c, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/cd3140b4967e46e88686729c0b59919f, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/aad8221e321945fe85605d38843ebb2f, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/07bbeefee46c4ce49cfa3f62b0f24b77, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/558d96c6f1314fa085eb857e0bd46a64, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/6bbbf9b699c94bda9f6911c90a4a0a1c, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/52fd63813fa64183b8a214d000fafd90, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/b1cbb8f055814c13bebebe2e711525e8, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/ea55d8de93be49139efeb4786d041393, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/4cc721003c4b4555a10eaf04b6254b5b, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/b2824d53988b417e8571cc9b390a7e29, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/894f216a2a9a41439070984dcab56a8d, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/5cb7baf496164698b2cd7e18d18acf12, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/604166de2dcd4e6fa3831797fdcae72b, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/9de6a23c0b334b6db88a00c9f7ac5aec, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/040d445c4b4d4d35b4ba9f3e6a6488fa, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/53c77c5f5dcc48daa33cac83f78f4558, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/143b9d9a469c4f97b6df3c6a802b50d4, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/0234f3af39da4767a907c20fbe367967, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/d945eb9f65964b37993c4e0f3e42c78c] to archive 2024-11-07T12:43:59,251 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T12:43:59,252 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/c7dee467bb744760bc636cc8a9f6ffb4 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/c7dee467bb744760bc636cc8a9f6ffb4 2024-11-07T12:43:59,253 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/09780355c6714c6e9a59d31ee86f729c to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/09780355c6714c6e9a59d31ee86f729c 2024-11-07T12:43:59,254 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/8e40ed468304427499086b35a1d5d70d to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/8e40ed468304427499086b35a1d5d70d 2024-11-07T12:43:59,254 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/c8022b066d3e49fcada23459efbcf950 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/c8022b066d3e49fcada23459efbcf950 2024-11-07T12:43:59,255 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/308142f3672b41cab2eeac0a6eecef38 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/308142f3672b41cab2eeac0a6eecef38 2024-11-07T12:43:59,256 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/ac18115502bd40fab50c1f0a689a6759 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/ac18115502bd40fab50c1f0a689a6759 2024-11-07T12:43:59,257 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/43d6ff7d297e4a8fa7ef199c86972bf2 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/43d6ff7d297e4a8fa7ef199c86972bf2 2024-11-07T12:43:59,257 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/ff11c6e977764ce585ca73a34b014061 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/ff11c6e977764ce585ca73a34b014061 2024-11-07T12:43:59,258 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/18951ee79df04451be85a8704467ede8 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/18951ee79df04451be85a8704467ede8 2024-11-07T12:43:59,259 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/60b7b3d60b3d41fa9914eaf7b178f683 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/60b7b3d60b3d41fa9914eaf7b178f683 2024-11-07T12:43:59,260 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/ad853594617743f9b3f8ccc4d4633d23 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/ad853594617743f9b3f8ccc4d4633d23 2024-11-07T12:43:59,260 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/656d2e3dca7a4df2ac13fce49e35dc97 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/656d2e3dca7a4df2ac13fce49e35dc97 2024-11-07T12:43:59,261 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/14da15b5f9be4df6827c69aa5b237cb2 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/14da15b5f9be4df6827c69aa5b237cb2 2024-11-07T12:43:59,262 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/40499930c9d44168bcb645af50ac9dfa to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/40499930c9d44168bcb645af50ac9dfa 2024-11-07T12:43:59,262 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/b9bcc05e42fb438f84da4b0fc99cda02 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/b9bcc05e42fb438f84da4b0fc99cda02 2024-11-07T12:43:59,263 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/c0a81589ffca484197b5a1c574e0132e to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/c0a81589ffca484197b5a1c574e0132e 2024-11-07T12:43:59,264 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/61c15cd61ec44297b2d8b03b23d0a19c to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/61c15cd61ec44297b2d8b03b23d0a19c 2024-11-07T12:43:59,265 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/f48af124758a4aebb4b22549fe26f479 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/f48af124758a4aebb4b22549fe26f479 2024-11-07T12:43:59,266 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/7fac4daa648d4ac9be32b290b192a61c to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/7fac4daa648d4ac9be32b290b192a61c 2024-11-07T12:43:59,266 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/cd3140b4967e46e88686729c0b59919f to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/cd3140b4967e46e88686729c0b59919f 2024-11-07T12:43:59,267 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/aad8221e321945fe85605d38843ebb2f to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/aad8221e321945fe85605d38843ebb2f 2024-11-07T12:43:59,268 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/07bbeefee46c4ce49cfa3f62b0f24b77 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/07bbeefee46c4ce49cfa3f62b0f24b77 2024-11-07T12:43:59,269 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/558d96c6f1314fa085eb857e0bd46a64 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/558d96c6f1314fa085eb857e0bd46a64 2024-11-07T12:43:59,270 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/6bbbf9b699c94bda9f6911c90a4a0a1c to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/6bbbf9b699c94bda9f6911c90a4a0a1c 2024-11-07T12:43:59,270 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/52fd63813fa64183b8a214d000fafd90 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/52fd63813fa64183b8a214d000fafd90 2024-11-07T12:43:59,271 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/b1cbb8f055814c13bebebe2e711525e8 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/b1cbb8f055814c13bebebe2e711525e8 2024-11-07T12:43:59,272 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/ea55d8de93be49139efeb4786d041393 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/ea55d8de93be49139efeb4786d041393 2024-11-07T12:43:59,273 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/4cc721003c4b4555a10eaf04b6254b5b to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/4cc721003c4b4555a10eaf04b6254b5b 2024-11-07T12:43:59,273 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/b2824d53988b417e8571cc9b390a7e29 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/b2824d53988b417e8571cc9b390a7e29 2024-11-07T12:43:59,274 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/894f216a2a9a41439070984dcab56a8d to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/894f216a2a9a41439070984dcab56a8d 2024-11-07T12:43:59,275 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/5cb7baf496164698b2cd7e18d18acf12 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/5cb7baf496164698b2cd7e18d18acf12 2024-11-07T12:43:59,275 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/604166de2dcd4e6fa3831797fdcae72b to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/604166de2dcd4e6fa3831797fdcae72b 2024-11-07T12:43:59,276 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/9de6a23c0b334b6db88a00c9f7ac5aec to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/9de6a23c0b334b6db88a00c9f7ac5aec 2024-11-07T12:43:59,277 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/040d445c4b4d4d35b4ba9f3e6a6488fa to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/040d445c4b4d4d35b4ba9f3e6a6488fa 2024-11-07T12:43:59,277 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/53c77c5f5dcc48daa33cac83f78f4558 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/53c77c5f5dcc48daa33cac83f78f4558 2024-11-07T12:43:59,278 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/143b9d9a469c4f97b6df3c6a802b50d4 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/143b9d9a469c4f97b6df3c6a802b50d4 2024-11-07T12:43:59,279 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/0234f3af39da4767a907c20fbe367967 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/0234f3af39da4767a907c20fbe367967 2024-11-07T12:43:59,280 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/d945eb9f65964b37993c4e0f3e42c78c to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/d945eb9f65964b37993c4e0f3e42c78c 2024-11-07T12:43:59,281 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/2360ab686d1a4ceba735eb4b44232082, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/b56fa0771a9a4a1d8591902acdf31104, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/022d206a82f54dd9ae5480872bf6da44, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/6c669c34daef441aae2ec433b04d2efd, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/3873223dbe5d4c49a920eab6632c61af, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/1f827ba940614c04b450a74cdef0b626, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/f0437c51a8ab4711b1b975bacd5a9414, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/7011b00d08b741509e02be8e21885370, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/d08906e3ea444fe6a662fc09b0c48b13, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/c42b46cdd0654982a3e3ab38107e54da, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/4cd28318f27c4b3893ca37805a6b557f, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/ca875722e7484770be1d900edc1e68d0, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/9b717e3fbf5c4fa19a794e88ae498682, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/23f837ded55744d5b6b513704f11a628, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/fa1148d0606c4a80ab3e7ad925a34da9, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/ce9dd77ac9334b91a5d159f6794939ca, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/c6eba27ff52c42778f7bfd90259b3051, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/7e3b4c8e9d9b4fa18df7c5e298a5b99b, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/2aa93a34157a413c844ee47a6d32c7c4, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/a0adde41ab0a48b0bcae60c5b473efde, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/be49a87cc52647efb51217fce97f85c3, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/011fa10c6f0f49c6a487f835a1ba6702, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/4cb385971a07498681662ecfecde0850, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/8963df44fc5f42adaea6d1f1226e8eef, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/6a2f1c616a914ad2b09bdf8244091ba8, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/ffdb01f0cb4f435d9eadf970cbcd0dd1, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/1c44bd143a7d433da3269f3b795bc0d4, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/840bafd3df7a4b998f1f45798349f4d0, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/71ddc6e133934c648245167de8d1feda, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/dd4f586e079c4913b8d78fa2476451eb, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/78f6b616cb314a1aa243f125e9498a9c, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/0761afa32d974e6d984049e1e5e617a8, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/183f04cfa6d8449e93fd160dbc88ac13, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/20b308226f694491aea04304eb607ef5, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/a68fbf4366ac44128e0bd3c16ea7ee8d, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/f62c93813fc9469c9e6537c401e5afda, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/b357060658cd41e38f55707835da2140, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/c8b9bc5f17f94aec8af0690635a56723, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/c8a7f581f8584b5686574c7df9cefc5f] to archive 2024-11-07T12:43:59,282 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T12:43:59,283 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/2360ab686d1a4ceba735eb4b44232082 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/2360ab686d1a4ceba735eb4b44232082 2024-11-07T12:43:59,283 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/b56fa0771a9a4a1d8591902acdf31104 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/b56fa0771a9a4a1d8591902acdf31104 2024-11-07T12:43:59,284 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/022d206a82f54dd9ae5480872bf6da44 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/022d206a82f54dd9ae5480872bf6da44 2024-11-07T12:43:59,285 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/6c669c34daef441aae2ec433b04d2efd to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/6c669c34daef441aae2ec433b04d2efd 2024-11-07T12:43:59,285 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/3873223dbe5d4c49a920eab6632c61af to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/3873223dbe5d4c49a920eab6632c61af 2024-11-07T12:43:59,286 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/1f827ba940614c04b450a74cdef0b626 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/1f827ba940614c04b450a74cdef0b626 2024-11-07T12:43:59,287 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/f0437c51a8ab4711b1b975bacd5a9414 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/f0437c51a8ab4711b1b975bacd5a9414 2024-11-07T12:43:59,288 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/7011b00d08b741509e02be8e21885370 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/7011b00d08b741509e02be8e21885370 2024-11-07T12:43:59,288 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/d08906e3ea444fe6a662fc09b0c48b13 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/d08906e3ea444fe6a662fc09b0c48b13 2024-11-07T12:43:59,289 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/c42b46cdd0654982a3e3ab38107e54da to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/c42b46cdd0654982a3e3ab38107e54da 2024-11-07T12:43:59,290 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/4cd28318f27c4b3893ca37805a6b557f to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/4cd28318f27c4b3893ca37805a6b557f 2024-11-07T12:43:59,291 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/ca875722e7484770be1d900edc1e68d0 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/ca875722e7484770be1d900edc1e68d0 2024-11-07T12:43:59,291 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/9b717e3fbf5c4fa19a794e88ae498682 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/9b717e3fbf5c4fa19a794e88ae498682 2024-11-07T12:43:59,292 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/23f837ded55744d5b6b513704f11a628 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/23f837ded55744d5b6b513704f11a628 2024-11-07T12:43:59,293 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/fa1148d0606c4a80ab3e7ad925a34da9 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/fa1148d0606c4a80ab3e7ad925a34da9 2024-11-07T12:43:59,293 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/ce9dd77ac9334b91a5d159f6794939ca to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/ce9dd77ac9334b91a5d159f6794939ca 2024-11-07T12:43:59,294 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/c6eba27ff52c42778f7bfd90259b3051 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/c6eba27ff52c42778f7bfd90259b3051 2024-11-07T12:43:59,295 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/7e3b4c8e9d9b4fa18df7c5e298a5b99b to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/7e3b4c8e9d9b4fa18df7c5e298a5b99b 2024-11-07T12:43:59,296 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/2aa93a34157a413c844ee47a6d32c7c4 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/2aa93a34157a413c844ee47a6d32c7c4 2024-11-07T12:43:59,296 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/a0adde41ab0a48b0bcae60c5b473efde to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/a0adde41ab0a48b0bcae60c5b473efde 2024-11-07T12:43:59,297 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/be49a87cc52647efb51217fce97f85c3 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/be49a87cc52647efb51217fce97f85c3 2024-11-07T12:43:59,298 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/011fa10c6f0f49c6a487f835a1ba6702 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/011fa10c6f0f49c6a487f835a1ba6702 2024-11-07T12:43:59,299 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/4cb385971a07498681662ecfecde0850 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/4cb385971a07498681662ecfecde0850 2024-11-07T12:43:59,300 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/8963df44fc5f42adaea6d1f1226e8eef to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/8963df44fc5f42adaea6d1f1226e8eef 2024-11-07T12:43:59,300 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/6a2f1c616a914ad2b09bdf8244091ba8 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/6a2f1c616a914ad2b09bdf8244091ba8 2024-11-07T12:43:59,301 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/ffdb01f0cb4f435d9eadf970cbcd0dd1 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/ffdb01f0cb4f435d9eadf970cbcd0dd1 2024-11-07T12:43:59,302 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/1c44bd143a7d433da3269f3b795bc0d4 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/1c44bd143a7d433da3269f3b795bc0d4 2024-11-07T12:43:59,302 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/840bafd3df7a4b998f1f45798349f4d0 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/840bafd3df7a4b998f1f45798349f4d0 2024-11-07T12:43:59,303 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/71ddc6e133934c648245167de8d1feda to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/71ddc6e133934c648245167de8d1feda 2024-11-07T12:43:59,305 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/dd4f586e079c4913b8d78fa2476451eb to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/dd4f586e079c4913b8d78fa2476451eb 2024-11-07T12:43:59,306 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/78f6b616cb314a1aa243f125e9498a9c to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/78f6b616cb314a1aa243f125e9498a9c 2024-11-07T12:43:59,307 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/0761afa32d974e6d984049e1e5e617a8 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/0761afa32d974e6d984049e1e5e617a8 2024-11-07T12:43:59,307 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/183f04cfa6d8449e93fd160dbc88ac13 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/183f04cfa6d8449e93fd160dbc88ac13 2024-11-07T12:43:59,308 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/20b308226f694491aea04304eb607ef5 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/20b308226f694491aea04304eb607ef5 2024-11-07T12:43:59,309 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/a68fbf4366ac44128e0bd3c16ea7ee8d to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/a68fbf4366ac44128e0bd3c16ea7ee8d 2024-11-07T12:43:59,310 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/f62c93813fc9469c9e6537c401e5afda to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/f62c93813fc9469c9e6537c401e5afda 2024-11-07T12:43:59,310 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/b357060658cd41e38f55707835da2140 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/b357060658cd41e38f55707835da2140 2024-11-07T12:43:59,311 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/c8b9bc5f17f94aec8af0690635a56723 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/c8b9bc5f17f94aec8af0690635a56723 2024-11-07T12:43:59,312 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/c8a7f581f8584b5686574c7df9cefc5f to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/c8a7f581f8584b5686574c7df9cefc5f 2024-11-07T12:43:59,314 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/4b7f7a338fd14bb3abe360f9b651c790, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/5377496dd313408cbba0cee4cd77166f, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/85f12dac2a8b486989909b9dc8ed1cf1, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/4a1497e1972d4c4e9ec13bb61613848f, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/7b971499c22e4654be525f40df839869, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/6b089f6269ed47d69fc023d6f626cd3e, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/e4d489060d7e41c8953a042e6f024561, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/3fd6ce875c6a4a39b58e15d863c0f517, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/3a5151d403bd42c683df1863703c8832, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/405c0221dfef42f9b42a6384fbb6acdc, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/8a0a74a3b90544c99771294d7a658410, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/4c78918acfa64990b1f4fc92fb5eb712, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/887547cba1734220851859342c3e8621, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/c5866565a00a41d8a527090f8406af1d, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/5633e60e0c514eebb7ff263aff810a31, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/4168fddf4c3b4621bbe51525802b1b02, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/e048a892172b46f78a4c0f67a3fcb5a5, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/ac55478ae4e4458b9b2d359948466f77, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/ee64f4b0d8784f1db37c13caab95fff4, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/5d1ddf827958462394b1e1fd3e323521, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/4e39228f309b4a119ce3fda53056ec5a, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/a1d98a94ed454b478ba1dd30c9c87505, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/607b0632f6ea4030b233ea5ba3c760be, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/e76b8ceadf3d4ab6a5f473d9d40210cf, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/f02c3e6aa67e467993be6b243d2ebeba, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/f1fc0d2300c54ca7a6eb8cc8f43b1bc8, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/7f841c78c2214715bd9d2fbd9c828285, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/59aca4f5d1644d24b712406f3362734f, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/87654538469d4acd8cc405dc88911f24, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/64bc26786f3a4f04b4aca537a947b438, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/d2eab486a149417aaefcd5bf7582c9f1, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/0fc56b1d51364298a349e0d2758ab3c3, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/e0220e56e81c4e47a34527d9dd8281b8, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/0a7639e03b3b43c78962d198da874092, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/93f792033b0f4644a8b5c8fa3be56607, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/eb60687b20ee43e5a0b3522b44456222, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/522e4a1167f847d09959a8390277076a, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/4bb818caaddb492d866e66f4a43312f4, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/f4286612e2cc4644b0a26b72e841037d, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/c3d2aa670a684b68a69fb2bb44076209] to archive 2024-11-07T12:43:59,314 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T12:43:59,315 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/4b7f7a338fd14bb3abe360f9b651c790 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/4b7f7a338fd14bb3abe360f9b651c790 2024-11-07T12:43:59,317 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/5377496dd313408cbba0cee4cd77166f to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/5377496dd313408cbba0cee4cd77166f 2024-11-07T12:43:59,318 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/85f12dac2a8b486989909b9dc8ed1cf1 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/85f12dac2a8b486989909b9dc8ed1cf1 2024-11-07T12:43:59,318 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/4a1497e1972d4c4e9ec13bb61613848f to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/4a1497e1972d4c4e9ec13bb61613848f 2024-11-07T12:43:59,319 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/7b971499c22e4654be525f40df839869 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/7b971499c22e4654be525f40df839869 2024-11-07T12:43:59,320 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/6b089f6269ed47d69fc023d6f626cd3e to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/6b089f6269ed47d69fc023d6f626cd3e 2024-11-07T12:43:59,321 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/e4d489060d7e41c8953a042e6f024561 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/e4d489060d7e41c8953a042e6f024561 2024-11-07T12:43:59,322 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/3fd6ce875c6a4a39b58e15d863c0f517 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/3fd6ce875c6a4a39b58e15d863c0f517 2024-11-07T12:43:59,322 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/3a5151d403bd42c683df1863703c8832 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/3a5151d403bd42c683df1863703c8832 2024-11-07T12:43:59,323 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/405c0221dfef42f9b42a6384fbb6acdc to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/405c0221dfef42f9b42a6384fbb6acdc 2024-11-07T12:43:59,324 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/8a0a74a3b90544c99771294d7a658410 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/8a0a74a3b90544c99771294d7a658410 2024-11-07T12:43:59,325 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/4c78918acfa64990b1f4fc92fb5eb712 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/4c78918acfa64990b1f4fc92fb5eb712 2024-11-07T12:43:59,326 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/887547cba1734220851859342c3e8621 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/887547cba1734220851859342c3e8621 2024-11-07T12:43:59,327 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/c5866565a00a41d8a527090f8406af1d to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/c5866565a00a41d8a527090f8406af1d 2024-11-07T12:43:59,327 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/5633e60e0c514eebb7ff263aff810a31 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/5633e60e0c514eebb7ff263aff810a31 2024-11-07T12:43:59,328 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/4168fddf4c3b4621bbe51525802b1b02 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/4168fddf4c3b4621bbe51525802b1b02 2024-11-07T12:43:59,329 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/e048a892172b46f78a4c0f67a3fcb5a5 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/e048a892172b46f78a4c0f67a3fcb5a5 2024-11-07T12:43:59,330 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/ac55478ae4e4458b9b2d359948466f77 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/ac55478ae4e4458b9b2d359948466f77 2024-11-07T12:43:59,331 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/ee64f4b0d8784f1db37c13caab95fff4 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/ee64f4b0d8784f1db37c13caab95fff4 2024-11-07T12:43:59,331 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/5d1ddf827958462394b1e1fd3e323521 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/5d1ddf827958462394b1e1fd3e323521 2024-11-07T12:43:59,332 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/4e39228f309b4a119ce3fda53056ec5a to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/4e39228f309b4a119ce3fda53056ec5a 2024-11-07T12:43:59,333 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/a1d98a94ed454b478ba1dd30c9c87505 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/a1d98a94ed454b478ba1dd30c9c87505 2024-11-07T12:43:59,334 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/607b0632f6ea4030b233ea5ba3c760be to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/607b0632f6ea4030b233ea5ba3c760be 2024-11-07T12:43:59,335 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/e76b8ceadf3d4ab6a5f473d9d40210cf to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/e76b8ceadf3d4ab6a5f473d9d40210cf 2024-11-07T12:43:59,335 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/f02c3e6aa67e467993be6b243d2ebeba to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/f02c3e6aa67e467993be6b243d2ebeba 2024-11-07T12:43:59,336 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/f1fc0d2300c54ca7a6eb8cc8f43b1bc8 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/f1fc0d2300c54ca7a6eb8cc8f43b1bc8 2024-11-07T12:43:59,337 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/7f841c78c2214715bd9d2fbd9c828285 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/7f841c78c2214715bd9d2fbd9c828285 2024-11-07T12:43:59,338 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/59aca4f5d1644d24b712406f3362734f to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/59aca4f5d1644d24b712406f3362734f 2024-11-07T12:43:59,338 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/87654538469d4acd8cc405dc88911f24 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/87654538469d4acd8cc405dc88911f24 2024-11-07T12:43:59,339 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/64bc26786f3a4f04b4aca537a947b438 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/64bc26786f3a4f04b4aca537a947b438 2024-11-07T12:43:59,340 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/d2eab486a149417aaefcd5bf7582c9f1 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/d2eab486a149417aaefcd5bf7582c9f1 2024-11-07T12:43:59,341 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/0fc56b1d51364298a349e0d2758ab3c3 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/0fc56b1d51364298a349e0d2758ab3c3 2024-11-07T12:43:59,341 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/e0220e56e81c4e47a34527d9dd8281b8 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/e0220e56e81c4e47a34527d9dd8281b8 2024-11-07T12:43:59,342 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/0a7639e03b3b43c78962d198da874092 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/0a7639e03b3b43c78962d198da874092 2024-11-07T12:43:59,343 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/93f792033b0f4644a8b5c8fa3be56607 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/93f792033b0f4644a8b5c8fa3be56607 2024-11-07T12:43:59,343 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/eb60687b20ee43e5a0b3522b44456222 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/eb60687b20ee43e5a0b3522b44456222 2024-11-07T12:43:59,344 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/522e4a1167f847d09959a8390277076a to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/522e4a1167f847d09959a8390277076a 2024-11-07T12:43:59,345 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/4bb818caaddb492d866e66f4a43312f4 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/4bb818caaddb492d866e66f4a43312f4 2024-11-07T12:43:59,346 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/f4286612e2cc4644b0a26b72e841037d to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/f4286612e2cc4644b0a26b72e841037d 2024-11-07T12:43:59,346 DEBUG [StoreCloser-TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/c3d2aa670a684b68a69fb2bb44076209 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/c3d2aa670a684b68a69fb2bb44076209 2024-11-07T12:43:59,350 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=189}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/recovered.edits/569.seqid, newMaxSeqId=569, maxSeqId=4 2024-11-07T12:43:59,350 INFO [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=189}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. 2024-11-07T12:43:59,351 DEBUG [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=189}] regionserver.HRegion(1676): Region close journal for 37eda6b0dad56caa0fa6c1ac8208d5b2: Waiting for close lock at 1730983437605Running coprocessor pre-close hooks at 1730983437605Disabling compacts and flushes for region at 1730983437605Disabling writes for close at 1730983437605Obtaining lock to block concurrent updates at 1730983437605Preparing flush snapshotting stores in 37eda6b0dad56caa0fa6c1ac8208d5b2 at 1730983437605Finished memstore snapshotting TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1730983437606 (+1 ms)Flushing stores of TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2. at 1730983437606Flushing 37eda6b0dad56caa0fa6c1ac8208d5b2/A: creating writer at 1730983437606Flushing mob file 37eda6b0dad56caa0fa6c1ac8208d5b2/A: appending metadata at 1730983437611 (+5 ms)Flushing mob file 37eda6b0dad56caa0fa6c1ac8208d5b2/A: closing flushed file at 1730983437611Flushing 37eda6b0dad56caa0fa6c1ac8208d5b2/A: appending metadata at 1730983438018 (+407 ms)Flushing 37eda6b0dad56caa0fa6c1ac8208d5b2/A: closing flushed file at 1730983438018Flushing 37eda6b0dad56caa0fa6c1ac8208d5b2/B: creating writer at 1730983438424 (+406 ms)Flushing 37eda6b0dad56caa0fa6c1ac8208d5b2/B: appending metadata at 1730983438426 (+2 ms)Flushing 37eda6b0dad56caa0fa6c1ac8208d5b2/B: closing flushed file at 1730983438426Flushing 37eda6b0dad56caa0fa6c1ac8208d5b2/C: creating writer at 1730983438832 (+406 ms)Flushing 37eda6b0dad56caa0fa6c1ac8208d5b2/C: appending metadata at 1730983438834 (+2 ms)Flushing 37eda6b0dad56caa0fa6c1ac8208d5b2/C: closing flushed file at 1730983438834Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@354a89b4: reopening flushed file at 1730983439240 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@31e79b82: reopening flushed file at 1730983439243 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@508a417d: reopening flushed file at 1730983439246 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 37eda6b0dad56caa0fa6c1ac8208d5b2 in 1645ms, sequenceid=566, compaction requested=true at 1730983439250 (+4 ms)Writing region close event to WAL at 1730983439348 (+98 ms)Running coprocessor post-close hooks at 1730983439350 (+2 ms)Closed at 1730983439350 2024-11-07T12:43:59,352 INFO [RS_CLOSE_REGION-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_REGION, pid=189}] handler.UnassignRegionHandler(157): Closed 37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:59,353 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=188 updating hbase:meta row=37eda6b0dad56caa0fa6c1ac8208d5b2, regionState=CLOSED 2024-11-07T12:43:59,354 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=189, ppid=188, state=RUNNABLE, hasLock=false; CloseRegionProcedure 37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 because future has completed 2024-11-07T12:43:59,355 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=189, resume processing ppid=188 2024-11-07T12:43:59,355 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=189, ppid=188, state=SUCCESS, hasLock=false; CloseRegionProcedure 37eda6b0dad56caa0fa6c1ac8208d5b2, server=db9ad1cb6cf9,34293,1730983260217 in 1.9010 sec 2024-11-07T12:43:59,357 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=188, resume processing ppid=187 2024-11-07T12:43:59,357 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=188, ppid=187, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=37eda6b0dad56caa0fa6c1ac8208d5b2, UNASSIGN in 1.9050 sec 2024-11-07T12:43:59,359 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=187, resume processing ppid=186 2024-11-07T12:43:59,359 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=187, ppid=186, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9080 sec 2024-11-07T12:43:59,360 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730983439360"}]},"ts":"1730983439360"} 2024-11-07T12:43:59,361 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-07T12:43:59,361 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-07T12:43:59,362 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=186, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9160 sec 2024-11-07T12:43:59,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=186 2024-11-07T12:43:59,591 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:43:59,592 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-07T12:43:59,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] procedure2.ProcedureExecutor(1139): Stored pid=190, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:43:59,594 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=190, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:43:59,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=190 2024-11-07T12:43:59,594 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=190, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:43:59,596 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:59,597 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A, FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B, FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C, FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/recovered.edits] 2024-11-07T12:43:59,599 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/0503931b799e40858c9dd08fa5fe1184 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/0503931b799e40858c9dd08fa5fe1184 2024-11-07T12:43:59,600 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/485dff2dbbf5467f8f34f85d2ee17d5d to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/485dff2dbbf5467f8f34f85d2ee17d5d 2024-11-07T12:43:59,601 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/a97d15783a1b438fbe9fea2030fd64a0 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/a97d15783a1b438fbe9fea2030fd64a0 2024-11-07T12:43:59,601 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/cb3064520f7b42d6ade3cb2186c1c18e to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/A/cb3064520f7b42d6ade3cb2186c1c18e 2024-11-07T12:43:59,603 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/9ea66c25743d415783583b4ca5d5be41 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/9ea66c25743d415783583b4ca5d5be41 2024-11-07T12:43:59,604 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/d74b76bfdf1441ffaa8fc792c68bb3dc to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/d74b76bfdf1441ffaa8fc792c68bb3dc 2024-11-07T12:43:59,604 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/e8d1a018faf141e997ca463c38f363c1 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/e8d1a018faf141e997ca463c38f363c1 2024-11-07T12:43:59,605 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/e9241c374f6d48ea94772e4a8d218719 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/B/e9241c374f6d48ea94772e4a8d218719 2024-11-07T12:43:59,607 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/b783d2f36e3142c9bb8164a5d00a1c9a to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/b783d2f36e3142c9bb8164a5d00a1c9a 2024-11-07T12:43:59,607 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/d3964a06491647979c15212fffebb987 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/d3964a06491647979c15212fffebb987 2024-11-07T12:43:59,608 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/f5d1e5e0cd054afe99950343f5093a81 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/C/f5d1e5e0cd054afe99950343f5093a81 2024-11-07T12:43:59,611 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/recovered.edits/569.seqid to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2/recovered.edits/569.seqid 2024-11-07T12:43:59,612 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/default/TestAcidGuarantees/37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:59,612 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-07T12:43:59,612 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-07T12:43:59,613 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-07T12:43:59,616 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411070113c49441a04189ae827673b54b0b45_37eda6b0dad56caa0fa6c1ac8208d5b2 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411070113c49441a04189ae827673b54b0b45_37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:59,617 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411070b73a5535b5e43779ccda789e039516b_37eda6b0dad56caa0fa6c1ac8208d5b2 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411070b73a5535b5e43779ccda789e039516b_37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:59,618 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411070d9738eba2254ff8991182ff91c4e8cc_37eda6b0dad56caa0fa6c1ac8208d5b2 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411070d9738eba2254ff8991182ff91c4e8cc_37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:59,619 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411070e79acc8681c417885eacbd1d7171735_37eda6b0dad56caa0fa6c1ac8208d5b2 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411070e79acc8681c417885eacbd1d7171735_37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:59,619 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107189390bbc7f5409fba767f607cdd3369_37eda6b0dad56caa0fa6c1ac8208d5b2 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107189390bbc7f5409fba767f607cdd3369_37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:59,620 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411071b9bad08d1cb4e5189e9f672d7927302_37eda6b0dad56caa0fa6c1ac8208d5b2 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411071b9bad08d1cb4e5189e9f672d7927302_37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:59,621 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110727011b0f09d2482088da5e31aa82d3b4_37eda6b0dad56caa0fa6c1ac8208d5b2 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110727011b0f09d2482088da5e31aa82d3b4_37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:59,622 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107309382900c0a4be4b931692b438ee0aa_37eda6b0dad56caa0fa6c1ac8208d5b2 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107309382900c0a4be4b931692b438ee0aa_37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:59,623 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411073322b56604ce4f54aaf66c858556ca4a_37eda6b0dad56caa0fa6c1ac8208d5b2 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411073322b56604ce4f54aaf66c858556ca4a_37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:59,623 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110737a83c12357e4e978b6278766ab49030_37eda6b0dad56caa0fa6c1ac8208d5b2 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110737a83c12357e4e978b6278766ab49030_37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:59,624 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411073b53cb1286dd4fb786b3898ff7619f77_37eda6b0dad56caa0fa6c1ac8208d5b2 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411073b53cb1286dd4fb786b3898ff7619f77_37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:59,625 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411073dd10d30d9de48a79a15f71ea471f970_37eda6b0dad56caa0fa6c1ac8208d5b2 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411073dd10d30d9de48a79a15f71ea471f970_37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:59,626 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107434f71175c7540fa99959fe426b9066b_37eda6b0dad56caa0fa6c1ac8208d5b2 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107434f71175c7540fa99959fe426b9066b_37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:59,626 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411074401f596178548a99548bc06acf057d3_37eda6b0dad56caa0fa6c1ac8208d5b2 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411074401f596178548a99548bc06acf057d3_37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:59,627 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107483524e7f47f402085e1e26abd777a88_37eda6b0dad56caa0fa6c1ac8208d5b2 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107483524e7f47f402085e1e26abd777a88_37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:59,628 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411074d48a49edfd74746bc1497fc056bff2d_37eda6b0dad56caa0fa6c1ac8208d5b2 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411074d48a49edfd74746bc1497fc056bff2d_37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:59,629 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107698199540cbf46fc83057360807867fc_37eda6b0dad56caa0fa6c1ac8208d5b2 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107698199540cbf46fc83057360807867fc_37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:59,630 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411076d0ddb3c23ac4612bd4708a8c04a467e_37eda6b0dad56caa0fa6c1ac8208d5b2 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411076d0ddb3c23ac4612bd4708a8c04a467e_37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:59,630 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110798adc6dfb99046f08cfda5cf6dff8203_37eda6b0dad56caa0fa6c1ac8208d5b2 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110798adc6dfb99046f08cfda5cf6dff8203_37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:59,631 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107b81f059cab444f298f11c51c5726594c_37eda6b0dad56caa0fa6c1ac8208d5b2 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107b81f059cab444f298f11c51c5726594c_37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:59,632 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107b9c10c8b522345fb8dfc9be947af040b_37eda6b0dad56caa0fa6c1ac8208d5b2 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107b9c10c8b522345fb8dfc9be947af040b_37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:59,633 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107bf62a513b19e4f6dace68e1e4312183d_37eda6b0dad56caa0fa6c1ac8208d5b2 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107bf62a513b19e4f6dace68e1e4312183d_37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:59,634 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107d2da596e1bb64ab7beb5bd88937be3e9_37eda6b0dad56caa0fa6c1ac8208d5b2 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107d2da596e1bb64ab7beb5bd88937be3e9_37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:59,634 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107d5f74e69f07b4d3d908a06cde799ecf5_37eda6b0dad56caa0fa6c1ac8208d5b2 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107d5f74e69f07b4d3d908a06cde799ecf5_37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:59,635 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107d7bd9db3043f41809b4e46f44954c807_37eda6b0dad56caa0fa6c1ac8208d5b2 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107d7bd9db3043f41809b4e46f44954c807_37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:59,636 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107db2ec0652cf746c5b3d1cf4311f3bc3e_37eda6b0dad56caa0fa6c1ac8208d5b2 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107db2ec0652cf746c5b3d1cf4311f3bc3e_37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:59,637 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107e17778132d1f4d629b28402a778d0677_37eda6b0dad56caa0fa6c1ac8208d5b2 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107e17778132d1f4d629b28402a778d0677_37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:59,638 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107e59d21d435d043139bc144ce9d23b99f_37eda6b0dad56caa0fa6c1ac8208d5b2 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107e59d21d435d043139bc144ce9d23b99f_37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:59,638 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107e909ba759e864dcda1c3d9625bab982c_37eda6b0dad56caa0fa6c1ac8208d5b2 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107e909ba759e864dcda1c3d9625bab982c_37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:59,639 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107f32715b3233e42598bd6632035723a3e_37eda6b0dad56caa0fa6c1ac8208d5b2 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107f32715b3233e42598bd6632035723a3e_37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:59,640 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107f902386e4a3c4e07bc76f2fad14b19f8_37eda6b0dad56caa0fa6c1ac8208d5b2 to hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107f902386e4a3c4e07bc76f2fad14b19f8_37eda6b0dad56caa0fa6c1ac8208d5b2 2024-11-07T12:43:59,640 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-07T12:43:59,642 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=190, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:43:59,644 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-07T12:43:59,645 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-07T12:43:59,646 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=190, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:43:59,646 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-07T12:43:59,646 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1730983439646"}]},"ts":"9223372036854775807"} 2024-11-07T12:43:59,648 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-07T12:43:59,648 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 37eda6b0dad56caa0fa6c1ac8208d5b2, NAME => 'TestAcidGuarantees,,1730983408352.37eda6b0dad56caa0fa6c1ac8208d5b2.', STARTKEY => '', ENDKEY => ''}] 2024-11-07T12:43:59,648 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-07T12:43:59,648 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1730983439648"}]},"ts":"9223372036854775807"} 2024-11-07T12:43:59,649 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-07T12:43:59,650 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=190, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:43:59,651 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=190, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 58 msec 2024-11-07T12:43:59,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=190 2024-11-07T12:43:59,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-07T12:43:59,702 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:43:59,710 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=246 (was 249), OpenFileDescriptor=449 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=662 (was 861), ProcessCount=11 (was 11), AvailableMemoryMB=2540 (was 2504) - AvailableMemoryMB LEAK? - 2024-11-07T12:43:59,710 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-07T12:43:59,711 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-07T12:43:59,711 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T12:43:59,711 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:43:59,711 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:43:59,711 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:43:59,711 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-07T12:43:59,711 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1258311727, stopped=false 2024-11-07T12:43:59,712 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=db9ad1cb6cf9,38619,1730983259119 2024-11-07T12:43:59,714 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34293-0x1001a4271e00001, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-07T12:43:59,714 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38619-0x1001a4271e00000, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-07T12:43:59,714 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34293-0x1001a4271e00001, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T12:43:59,714 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38619-0x1001a4271e00000, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T12:43:59,714 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T12:43:59,715 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34293-0x1001a4271e00001, quorum=127.0.0.1:49805, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T12:43:59,715 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-07T12:43:59,715 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:38619-0x1001a4271e00000, quorum=127.0.0.1:49805, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T12:43:59,715 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T12:43:59,715 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:43:59,715 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'db9ad1cb6cf9,34293,1730983260217' ***** 2024-11-07T12:43:59,716 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-07T12:43:59,716 INFO [RS:0;db9ad1cb6cf9:34293 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-07T12:43:59,716 INFO [RS:0;db9ad1cb6cf9:34293 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-07T12:43:59,716 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-07T12:43:59,716 INFO [RS:0;db9ad1cb6cf9:34293 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-07T12:43:59,717 INFO [RS:0;db9ad1cb6cf9:34293 {}] regionserver.HRegionServer(959): stopping server db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:43:59,717 INFO [RS:0;db9ad1cb6cf9:34293 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-07T12:43:59,717 INFO [RS:0;db9ad1cb6cf9:34293 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;db9ad1cb6cf9:34293. 2024-11-07T12:43:59,717 DEBUG [RS:0;db9ad1cb6cf9:34293 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T12:43:59,717 DEBUG [RS:0;db9ad1cb6cf9:34293 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:43:59,717 INFO [RS:0;db9ad1cb6cf9:34293 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-07T12:43:59,717 INFO [RS:0;db9ad1cb6cf9:34293 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-07T12:43:59,717 INFO [RS:0;db9ad1cb6cf9:34293 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-07T12:43:59,717 INFO [RS:0;db9ad1cb6cf9:34293 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-07T12:43:59,718 INFO [RS:0;db9ad1cb6cf9:34293 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-07T12:43:59,718 DEBUG [RS:0;db9ad1cb6cf9:34293 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-07T12:43:59,718 DEBUG [RS_CLOSE_META-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-07T12:43:59,718 INFO [RS_CLOSE_META-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-07T12:43:59,718 DEBUG [RS_CLOSE_META-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-07T12:43:59,718 DEBUG [RS:0;db9ad1cb6cf9:34293 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-07T12:43:59,718 DEBUG [RS_CLOSE_META-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-07T12:43:59,718 DEBUG [RS_CLOSE_META-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-07T12:43:59,718 INFO [RS_CLOSE_META-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-07T12:43:59,718 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-07T12:43:59,742 DEBUG [RS_CLOSE_META-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/hbase/meta/1588230740/.tmp/info/89c88b8497eb45ceb3bacd811f438cb4 is 95, key is TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe./info:/1730983296584/DeleteFamily/seqid=0 2024-11-07T12:43:59,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742652_1828 (size=6514) 2024-11-07T12:43:59,786 INFO [regionserver/db9ad1cb6cf9:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-07T12:43:59,918 DEBUG [RS:0;db9ad1cb6cf9:34293 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-07T12:44:00,118 DEBUG [RS:0;db9ad1cb6cf9:34293 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-07T12:44:00,146 INFO [RS_CLOSE_META-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/hbase/meta/1588230740/.tmp/info/89c88b8497eb45ceb3bacd811f438cb4 2024-11-07T12:44:00,163 DEBUG [RS_CLOSE_META-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/hbase/meta/1588230740/.tmp/ns/23718791fb684bb1b204ce621a48bfdc is 93, key is TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe./ns:/1730983296522/DeleteFamily/seqid=0 2024-11-07T12:44:00,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742653_1829 (size=5906) 2024-11-07T12:44:00,319 DEBUG [RS:0;db9ad1cb6cf9:34293 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-07T12:44:00,519 DEBUG [RS:0;db9ad1cb6cf9:34293 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-07T12:44:00,567 INFO [RS_CLOSE_META-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/hbase/meta/1588230740/.tmp/ns/23718791fb684bb1b204ce621a48bfdc 2024-11-07T12:44:00,584 DEBUG [RS_CLOSE_META-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/hbase/meta/1588230740/.tmp/rep_barrier/b8244e0c5ba64a50ba4b71474207fa21 is 102, key is TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe./rep_barrier:/1730983296522/DeleteFamily/seqid=0 2024-11-07T12:44:00,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742654_1830 (size=6025) 2024-11-07T12:44:00,681 INFO [regionserver/db9ad1cb6cf9:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-07T12:44:00,681 INFO [regionserver/db9ad1cb6cf9:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-07T12:44:00,719 DEBUG [RS:0;db9ad1cb6cf9:34293 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-07T12:44:00,919 DEBUG [RS:0;db9ad1cb6cf9:34293 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-07T12:44:00,987 INFO [RS_CLOSE_META-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/hbase/meta/1588230740/.tmp/rep_barrier/b8244e0c5ba64a50ba4b71474207fa21 2024-11-07T12:44:01,004 DEBUG [RS_CLOSE_META-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/hbase/meta/1588230740/.tmp/table/16c17652b17d45bfa24bb8b2aab8ce66 is 96, key is TestAcidGuarantees,,1730983264222.13f50260f7a437ef753d61fda36692fe./table:/1730983296522/DeleteFamily/seqid=0 2024-11-07T12:44:01,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742655_1831 (size=5936) 2024-11-07T12:44:01,119 DEBUG [RS:0;db9ad1cb6cf9:34293 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-07T12:44:01,320 DEBUG [RS:0;db9ad1cb6cf9:34293 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-07T12:44:01,408 INFO [RS_CLOSE_META-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/hbase/meta/1588230740/.tmp/table/16c17652b17d45bfa24bb8b2aab8ce66 2024-11-07T12:44:01,411 DEBUG [RS_CLOSE_META-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/hbase/meta/1588230740/.tmp/info/89c88b8497eb45ceb3bacd811f438cb4 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/hbase/meta/1588230740/info/89c88b8497eb45ceb3bacd811f438cb4 2024-11-07T12:44:01,413 INFO [RS_CLOSE_META-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/hbase/meta/1588230740/info/89c88b8497eb45ceb3bacd811f438cb4, entries=12, sequenceid=90, filesize=6.4 K 2024-11-07T12:44:01,414 DEBUG [RS_CLOSE_META-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/hbase/meta/1588230740/.tmp/ns/23718791fb684bb1b204ce621a48bfdc as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/hbase/meta/1588230740/ns/23718791fb684bb1b204ce621a48bfdc 2024-11-07T12:44:01,416 INFO [RS_CLOSE_META-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/hbase/meta/1588230740/ns/23718791fb684bb1b204ce621a48bfdc, entries=8, sequenceid=90, filesize=5.8 K 2024-11-07T12:44:01,417 DEBUG [RS_CLOSE_META-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/hbase/meta/1588230740/.tmp/rep_barrier/b8244e0c5ba64a50ba4b71474207fa21 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/hbase/meta/1588230740/rep_barrier/b8244e0c5ba64a50ba4b71474207fa21 2024-11-07T12:44:01,419 INFO [RS_CLOSE_META-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/hbase/meta/1588230740/rep_barrier/b8244e0c5ba64a50ba4b71474207fa21, entries=6, sequenceid=90, filesize=5.9 K 2024-11-07T12:44:01,419 DEBUG [RS_CLOSE_META-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/hbase/meta/1588230740/.tmp/table/16c17652b17d45bfa24bb8b2aab8ce66 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/hbase/meta/1588230740/table/16c17652b17d45bfa24bb8b2aab8ce66 2024-11-07T12:44:01,422 INFO [RS_CLOSE_META-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/hbase/meta/1588230740/table/16c17652b17d45bfa24bb8b2aab8ce66, entries=7, sequenceid=90, filesize=5.8 K 2024-11-07T12:44:01,422 INFO [RS_CLOSE_META-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1704ms, sequenceid=90, compaction requested=false 2024-11-07T12:44:01,426 DEBUG [RS_CLOSE_META-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-07T12:44:01,426 DEBUG [RS_CLOSE_META-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-07T12:44:01,426 INFO [RS_CLOSE_META-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-07T12:44:01,427 DEBUG [RS_CLOSE_META-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1730983439718Running coprocessor pre-close hooks at 1730983439718Disabling compacts and flushes for region at 1730983439718Disabling writes for close at 1730983439718Obtaining lock to block concurrent updates at 1730983439718Preparing flush snapshotting stores in 1588230740 at 1730983439718Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1730983439719 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1730983439719Flushing 1588230740/info: creating writer at 1730983439720 (+1 ms)Flushing 1588230740/info: appending metadata at 1730983439742 (+22 ms)Flushing 1588230740/info: closing flushed file at 1730983439742Flushing 1588230740/ns: creating writer at 1730983440149 (+407 ms)Flushing 1588230740/ns: appending metadata at 1730983440163 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1730983440163Flushing 1588230740/rep_barrier: creating writer at 1730983440569 (+406 ms)Flushing 1588230740/rep_barrier: appending metadata at 1730983440583 (+14 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1730983440583Flushing 1588230740/table: creating writer at 1730983440990 (+407 ms)Flushing 1588230740/table: appending metadata at 1730983441004 (+14 ms)Flushing 1588230740/table: closing flushed file at 1730983441004Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@185a4385: reopening flushed file at 1730983441410 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5aae04f1: reopening flushed file at 1730983441414 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@672ba343: reopening flushed file at 1730983441416 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4c0a07f8: reopening flushed file at 1730983441419 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1704ms, sequenceid=90, compaction requested=false at 1730983441422 (+3 ms)Writing region close event to WAL at 1730983441424 (+2 ms)Running coprocessor post-close hooks at 1730983441426 (+2 ms)Closed at 1730983441426 2024-11-07T12:44:01,427 DEBUG [RS_CLOSE_META-regionserver/db9ad1cb6cf9:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-07T12:44:01,520 INFO [RS:0;db9ad1cb6cf9:34293 {}] regionserver.HRegionServer(976): stopping server db9ad1cb6cf9,34293,1730983260217; all regions closed. 2024-11-07T12:44:01,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741834_1010 (size=25344) 2024-11-07T12:44:01,526 DEBUG [RS:0;db9ad1cb6cf9:34293 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/oldWALs 2024-11-07T12:44:01,526 INFO [RS:0;db9ad1cb6cf9:34293 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL db9ad1cb6cf9%2C34293%2C1730983260217.meta:.meta(num 1730983263230) 2024-11-07T12:44:01,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741833_1009 (size=20283135) 2024-11-07T12:44:01,529 DEBUG [RS:0;db9ad1cb6cf9:34293 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/oldWALs 2024-11-07T12:44:01,529 INFO [RS:0;db9ad1cb6cf9:34293 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL db9ad1cb6cf9%2C34293%2C1730983260217:(num 1730983262782) 2024-11-07T12:44:01,529 DEBUG [RS:0;db9ad1cb6cf9:34293 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:44:01,529 INFO [RS:0;db9ad1cb6cf9:34293 {}] regionserver.LeaseManager(133): Closed leases 2024-11-07T12:44:01,530 INFO [RS:0;db9ad1cb6cf9:34293 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-07T12:44:01,530 INFO [RS:0;db9ad1cb6cf9:34293 {}] hbase.ChoreService(370): Chore service for: regionserver/db9ad1cb6cf9:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-07T12:44:01,530 INFO [RS:0;db9ad1cb6cf9:34293 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-07T12:44:01,530 INFO [regionserver/db9ad1cb6cf9:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-07T12:44:01,531 INFO [RS:0;db9ad1cb6cf9:34293 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34293 2024-11-07T12:44:01,535 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38619-0x1001a4271e00000, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-07T12:44:01,535 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34293-0x1001a4271e00001, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/db9ad1cb6cf9,34293,1730983260217 2024-11-07T12:44:01,535 INFO [RS:0;db9ad1cb6cf9:34293 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-07T12:44:01,537 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [db9ad1cb6cf9,34293,1730983260217] 2024-11-07T12:44:01,539 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/db9ad1cb6cf9,34293,1730983260217 already deleted, retry=false 2024-11-07T12:44:01,539 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; db9ad1cb6cf9,34293,1730983260217 expired; onlineServers=0 2024-11-07T12:44:01,539 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'db9ad1cb6cf9,38619,1730983259119' ***** 2024-11-07T12:44:01,539 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-07T12:44:01,539 INFO [M:0;db9ad1cb6cf9:38619 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-07T12:44:01,539 INFO [M:0;db9ad1cb6cf9:38619 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-07T12:44:01,539 DEBUG [M:0;db9ad1cb6cf9:38619 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-07T12:44:01,539 DEBUG [M:0;db9ad1cb6cf9:38619 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-07T12:44:01,539 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-07T12:44:01,539 DEBUG [master/db9ad1cb6cf9:0:becomeActiveMaster-HFileCleaner.large.0-1730983262249 {}] cleaner.HFileCleaner(306): Exit Thread[master/db9ad1cb6cf9:0:becomeActiveMaster-HFileCleaner.large.0-1730983262249,5,FailOnTimeoutGroup] 2024-11-07T12:44:01,539 DEBUG [master/db9ad1cb6cf9:0:becomeActiveMaster-HFileCleaner.small.0-1730983262253 {}] cleaner.HFileCleaner(306): Exit Thread[master/db9ad1cb6cf9:0:becomeActiveMaster-HFileCleaner.small.0-1730983262253,5,FailOnTimeoutGroup] 2024-11-07T12:44:01,540 INFO [M:0;db9ad1cb6cf9:38619 {}] hbase.ChoreService(370): Chore service for: master/db9ad1cb6cf9:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-07T12:44:01,540 INFO [M:0;db9ad1cb6cf9:38619 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-07T12:44:01,540 DEBUG [M:0;db9ad1cb6cf9:38619 {}] master.HMaster(1795): Stopping service threads 2024-11-07T12:44:01,540 INFO [M:0;db9ad1cb6cf9:38619 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-07T12:44:01,540 INFO [M:0;db9ad1cb6cf9:38619 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T12:44:01,540 ERROR [M:0;db9ad1cb6cf9:38619 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-07T12:44:01,541 INFO [M:0;db9ad1cb6cf9:38619 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-07T12:44:01,541 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-07T12:44:01,543 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38619-0x1001a4271e00000, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-07T12:44:01,543 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38619-0x1001a4271e00000, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T12:44:01,543 DEBUG [M:0;db9ad1cb6cf9:38619 {}] zookeeper.ZKUtil(347): master:38619-0x1001a4271e00000, quorum=127.0.0.1:49805, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-07T12:44:01,543 WARN [M:0;db9ad1cb6cf9:38619 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-07T12:44:01,543 INFO [M:0;db9ad1cb6cf9:38619 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/.lastflushedseqids 2024-11-07T12:44:01,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742656_1832 (size=72) 2024-11-07T12:44:01,638 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34293-0x1001a4271e00001, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T12:44:01,638 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34293-0x1001a4271e00001, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T12:44:01,638 INFO [RS:0;db9ad1cb6cf9:34293 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-07T12:44:01,638 INFO [RS:0;db9ad1cb6cf9:34293 {}] regionserver.HRegionServer(1031): Exiting; stopping=db9ad1cb6cf9,34293,1730983260217; zookeeper connection closed. 2024-11-07T12:44:01,639 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2a4dc73e {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2a4dc73e 2024-11-07T12:44:01,639 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-07T12:44:01,954 INFO [M:0;db9ad1cb6cf9:38619 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-07T12:44:01,954 INFO [M:0;db9ad1cb6cf9:38619 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-07T12:44:01,954 DEBUG [M:0;db9ad1cb6cf9:38619 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-07T12:44:01,954 INFO [M:0;db9ad1cb6cf9:38619 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T12:44:01,954 DEBUG [M:0;db9ad1cb6cf9:38619 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T12:44:01,954 DEBUG [M:0;db9ad1cb6cf9:38619 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-07T12:44:01,954 DEBUG [M:0;db9ad1cb6cf9:38619 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T12:44:01,954 INFO [M:0;db9ad1cb6cf9:38619 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=832.42 KB heapSize=1023.05 KB 2024-11-07T12:44:01,970 DEBUG [M:0;db9ad1cb6cf9:38619 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/110fae5ec41b4be5ad5cdbf15cb2a4f1 is 82, key is hbase:meta,,1/info:regioninfo/1730983263481/Put/seqid=0 2024-11-07T12:44:01,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742657_1833 (size=5672) 2024-11-07T12:44:02,373 INFO [M:0;db9ad1cb6cf9:38619 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2356 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/110fae5ec41b4be5ad5cdbf15cb2a4f1 2024-11-07T12:44:02,393 DEBUG [M:0;db9ad1cb6cf9:38619 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/be3dd5b23d464478bfd09ce669fc70d8 is 2481, key is \x00\x00\x00\x00\x00\x00\x00\x99/proc:d/1730983411050/Put/seqid=0 2024-11-07T12:44:02,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742658_1834 (size=48930) 2024-11-07T12:44:02,797 INFO [M:0;db9ad1cb6cf9:38619 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=831.86 KB at sequenceid=2356 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/be3dd5b23d464478bfd09ce669fc70d8 2024-11-07T12:44:02,800 INFO [M:0;db9ad1cb6cf9:38619 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for be3dd5b23d464478bfd09ce669fc70d8 2024-11-07T12:44:02,814 DEBUG [M:0;db9ad1cb6cf9:38619 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9f073f79ee134690a281ab04c775958b is 69, key is db9ad1cb6cf9,34293,1730983260217/rs:state/1730983262441/Put/seqid=0 2024-11-07T12:44:02,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073742659_1835 (size=5156) 2024-11-07T12:44:03,218 INFO [M:0;db9ad1cb6cf9:38619 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2356 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9f073f79ee134690a281ab04c775958b 2024-11-07T12:44:03,221 DEBUG [M:0;db9ad1cb6cf9:38619 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/110fae5ec41b4be5ad5cdbf15cb2a4f1 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/110fae5ec41b4be5ad5cdbf15cb2a4f1 2024-11-07T12:44:03,223 INFO [M:0;db9ad1cb6cf9:38619 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/110fae5ec41b4be5ad5cdbf15cb2a4f1, entries=8, sequenceid=2356, filesize=5.5 K 2024-11-07T12:44:03,224 DEBUG [M:0;db9ad1cb6cf9:38619 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/be3dd5b23d464478bfd09ce669fc70d8 as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/be3dd5b23d464478bfd09ce669fc70d8 2024-11-07T12:44:03,226 INFO [M:0;db9ad1cb6cf9:38619 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for be3dd5b23d464478bfd09ce669fc70d8 2024-11-07T12:44:03,226 INFO [M:0;db9ad1cb6cf9:38619 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/be3dd5b23d464478bfd09ce669fc70d8, entries=190, sequenceid=2356, filesize=47.8 K 2024-11-07T12:44:03,226 DEBUG [M:0;db9ad1cb6cf9:38619 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9f073f79ee134690a281ab04c775958b as hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/9f073f79ee134690a281ab04c775958b 2024-11-07T12:44:03,229 INFO [M:0;db9ad1cb6cf9:38619 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/a053c693-45a1-785d-a6c3-a111cfef99c4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/9f073f79ee134690a281ab04c775958b, entries=1, sequenceid=2356, filesize=5.0 K 2024-11-07T12:44:03,229 INFO [M:0;db9ad1cb6cf9:38619 {}] regionserver.HRegion(3140): Finished flush of dataSize ~832.42 KB/852394, heapSize ~1022.75 KB/1047296, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1275ms, sequenceid=2356, compaction requested=false 2024-11-07T12:44:03,230 INFO [M:0;db9ad1cb6cf9:38619 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T12:44:03,231 DEBUG [M:0;db9ad1cb6cf9:38619 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1730983441954Disabling compacts and flushes for region at 1730983441954Disabling writes for close at 1730983441954Obtaining lock to block concurrent updates at 1730983441954Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1730983441954Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=852394, getHeapSize=1047536, getOffHeapSize=0, getCellsCount=2571 at 1730983441955 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1730983441955Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1730983441955Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1730983441970 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1730983441970Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1730983442376 (+406 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1730983442393 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1730983442393Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1730983442800 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1730983442814 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1730983442814Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@38082c64: reopening flushed file at 1730983443220 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2f6ff43c: reopening flushed file at 1730983443223 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@20e9eea6: reopening flushed file at 1730983443226 (+3 ms)Finished flush of dataSize ~832.42 KB/852394, heapSize ~1022.75 KB/1047296, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1275ms, sequenceid=2356, compaction requested=false at 1730983443229 (+3 ms)Writing region close event to WAL at 1730983443230 (+1 ms)Closed at 1730983443230 2024-11-07T12:44:03,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36063 is added to blk_1073741830_1006 (size=1006813) 2024-11-07T12:44:03,233 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-07T12:44:03,233 INFO [M:0;db9ad1cb6cf9:38619 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-07T12:44:03,234 INFO [M:0;db9ad1cb6cf9:38619 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38619 2024-11-07T12:44:03,234 INFO [M:0;db9ad1cb6cf9:38619 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-07T12:44:03,336 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38619-0x1001a4271e00000, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T12:44:03,336 INFO [M:0;db9ad1cb6cf9:38619 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-07T12:44:03,336 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38619-0x1001a4271e00000, quorum=127.0.0.1:49805, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T12:44:03,341 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-07T12:44:03,343 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-07T12:44:03,343 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-07T12:44:03,343 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-07T12:44:03,343 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ba7e58d5-de0c-7392-0264-ead072f99c84/hadoop.log.dir/,STOPPED} 2024-11-07T12:44:03,346 WARN [BP-138800792-172.17.0.2-1730983252949 heartbeating to localhost/127.0.0.1:37717 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-07T12:44:03,346 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-07T12:44:03,346 WARN [BP-138800792-172.17.0.2-1730983252949 heartbeating to localhost/127.0.0.1:37717 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-138800792-172.17.0.2-1730983252949 (Datanode Uuid bea1d833-e7f4-4776-b040-860a71f94e0f) service to localhost/127.0.0.1:37717 2024-11-07T12:44:03,346 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-07T12:44:03,348 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ba7e58d5-de0c-7392-0264-ead072f99c84/cluster_73ed722e-8252-77e5-7683-f63666bd3476/data/data1/current/BP-138800792-172.17.0.2-1730983252949 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-07T12:44:03,348 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ba7e58d5-de0c-7392-0264-ead072f99c84/cluster_73ed722e-8252-77e5-7683-f63666bd3476/data/data2/current/BP-138800792-172.17.0.2-1730983252949 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-07T12:44:03,349 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-07T12:44:03,356 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-07T12:44:03,356 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-07T12:44:03,356 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-07T12:44:03,357 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-07T12:44:03,357 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ba7e58d5-de0c-7392-0264-ead072f99c84/hadoop.log.dir/,STOPPED} 2024-11-07T12:44:03,374 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-07T12:44:03,522 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down